diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs10:obs15:stride4:seed10/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs10:obs15:stride4:seed10/config.json new file mode 100644 index 0000000000000000000000000000000000000000..dac54e1f3cf6bd91c5df552313d7b4ed6516eff6 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs10:obs15:stride4:seed10/config.json @@ -0,0 +1,255 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_frame_stack_uniform_u2_6_fs10_seed10", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 10, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-fs10-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "frame_stack", + "uniform", + "uniform_u2_6", + "fs10", + "seed10" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 15.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 10, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 10, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs10_seed10/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_frame_stack_uniform_u2_6_fs10_seed10 --train_dir results/checkpoints_factor_sweeps/demon_attack/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 10 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs10_seed10/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 10 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 36000 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-fs10-uniform_u2_6 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack frame_stack uniform uniform_u2_6 fs10 seed10 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 15.0 --screen_size 84 --noop_max 30 --frame_stack 10 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_frame_stack_uniform_u2_6_fs10_seed10", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 10, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-fs10-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "frame_stack", + "uniform", + "uniform_u2_6", + "fs10", + "seed10" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 15.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 10, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 10, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs10_seed10/episode_metrics.jsonl" + }, + "git_hash": "eb3a2e1efbd2aa03a60d7f44f5e18d8fdd0f5a2d", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/context_window/train/frame_stack/uniform_u2_6/fs10/seed_10", + "wandb_unique_id": "demon-attack-fs10-uniform_u2_6-s10" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs10:obs15:stride4:seed11/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs10:obs15:stride4:seed11/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..d240f5df6ae4ffe38854a1f42500effdf566568e --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs10:obs15:stride4:seed11/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a7e704d2345337c461e445ee123e1b6b44816a392b27a44de523c2c484b13262 +size 7937341 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs10:obs15:stride4:seed11/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs10:obs15:stride4:seed11/config.json new file mode 100644 index 0000000000000000000000000000000000000000..afbd677bb2125cbde2a742baf81e8c2bba7a67e8 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs10:obs15:stride4:seed11/config.json @@ -0,0 +1,255 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_frame_stack_uniform_u2_6_fs10_seed11", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 11, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-fs10-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "frame_stack", + "uniform", + "uniform_u2_6", + "fs10", + "seed11" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 15.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 10, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 11, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs10_seed11/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_frame_stack_uniform_u2_6_fs10_seed11 --train_dir results/checkpoints_factor_sweeps/demon_attack/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 11 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs10_seed11/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 11 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 36000 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-fs10-uniform_u2_6 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack frame_stack uniform uniform_u2_6 fs10 seed11 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 15.0 --screen_size 84 --noop_max 30 --frame_stack 10 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_frame_stack_uniform_u2_6_fs10_seed11", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 11, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-fs10-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "frame_stack", + "uniform", + "uniform_u2_6", + "fs10", + "seed11" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 15.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 10, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 11, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs10_seed11/episode_metrics.jsonl" + }, + "git_hash": "eb3a2e1efbd2aa03a60d7f44f5e18d8fdd0f5a2d", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/context_window/train/frame_stack/uniform_u2_6/fs10/seed_11", + "wandb_unique_id": "demon-attack-fs10-uniform_u2_6-s11" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs10:obs15:stride4:seed12/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs10:obs15:stride4:seed12/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..4b97455debaeea5612cbd48bddc2aa5cdd669bf5 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs10:obs15:stride4:seed12/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aa3ba64b1c0004383736d0ab9053c93331c6c5acbdf2d77f921be40e4769d294 +size 7937341 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs10:obs15:stride4:seed12/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs10:obs15:stride4:seed12/config.json new file mode 100644 index 0000000000000000000000000000000000000000..6edcf4470c7a230d07f5bb18c0ad5e3ba5f6b698 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs10:obs15:stride4:seed12/config.json @@ -0,0 +1,255 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_frame_stack_uniform_u2_6_fs10_seed12", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 12, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-fs10-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "frame_stack", + "uniform", + "uniform_u2_6", + "fs10", + "seed12" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 15.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 10, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 12, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs10_seed12/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_frame_stack_uniform_u2_6_fs10_seed12 --train_dir results/checkpoints_factor_sweeps/demon_attack/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 12 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs10_seed12/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 12 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 36000 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-fs10-uniform_u2_6 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack frame_stack uniform uniform_u2_6 fs10 seed12 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 15.0 --screen_size 84 --noop_max 30 --frame_stack 10 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_frame_stack_uniform_u2_6_fs10_seed12", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 12, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-fs10-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "frame_stack", + "uniform", + "uniform_u2_6", + "fs10", + "seed12" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 15.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 10, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 12, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs10_seed12/episode_metrics.jsonl" + }, + "git_hash": "eb3a2e1efbd2aa03a60d7f44f5e18d8fdd0f5a2d", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/context_window/train/frame_stack/uniform_u2_6/fs10/seed_12", + "wandb_unique_id": "demon-attack-fs10-uniform_u2_6-s12" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs10:obs15:stride4:seed13/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs10:obs15:stride4:seed13/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..67ae5944935f20e3304e6f6c3dc2888c08774c96 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs10:obs15:stride4:seed13/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:64a4f25efe6ac43fc878f2b6787da671a7b573c90992acbefe03d7b91283e62c +size 7937341 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs10:obs15:stride4:seed13/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs10:obs15:stride4:seed13/config.json new file mode 100644 index 0000000000000000000000000000000000000000..1cbbbbf65ecffa748b79479e1987a7875aaca112 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs10:obs15:stride4:seed13/config.json @@ -0,0 +1,255 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_frame_stack_uniform_u2_6_fs10_seed13", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 13, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-fs10-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "frame_stack", + "uniform", + "uniform_u2_6", + "fs10", + "seed13" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 15.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 10, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 13, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs10_seed13/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_frame_stack_uniform_u2_6_fs10_seed13 --train_dir results/checkpoints_factor_sweeps/demon_attack/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 13 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs10_seed13/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 13 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 36000 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-fs10-uniform_u2_6 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack frame_stack uniform uniform_u2_6 fs10 seed13 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 15.0 --screen_size 84 --noop_max 30 --frame_stack 10 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_frame_stack_uniform_u2_6_fs10_seed13", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 13, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-fs10-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "frame_stack", + "uniform", + "uniform_u2_6", + "fs10", + "seed13" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 15.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 10, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 13, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs10_seed13/episode_metrics.jsonl" + }, + "git_hash": "eb3a2e1efbd2aa03a60d7f44f5e18d8fdd0f5a2d", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/context_window/train/frame_stack/uniform_u2_6/fs10/seed_13", + "wandb_unique_id": "demon-attack-fs10-uniform_u2_6-s13" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs10:obs15:stride4:seed14/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs10:obs15:stride4:seed14/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..9f9ea8cc104683ab5aedc9a5ee9cec769a00b182 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs10:obs15:stride4:seed14/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8e0cbb7fff9c3b356151914f565d0e488601604c2d476076b98e386b8d5b358c +size 7937341 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs10:obs15:stride4:seed14/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs10:obs15:stride4:seed14/config.json new file mode 100644 index 0000000000000000000000000000000000000000..c845711d535a3b8c7af7b2133ffbdd625a9a00f0 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs10:obs15:stride4:seed14/config.json @@ -0,0 +1,255 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_frame_stack_uniform_u2_6_fs10_seed14", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 14, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-fs10-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "frame_stack", + "uniform", + "uniform_u2_6", + "fs10", + "seed14" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 15.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 10, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 14, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs10_seed14/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_frame_stack_uniform_u2_6_fs10_seed14 --train_dir results/checkpoints_factor_sweeps/demon_attack/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 14 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs10_seed14/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 14 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 36000 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-fs10-uniform_u2_6 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack frame_stack uniform uniform_u2_6 fs10 seed14 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 15.0 --screen_size 84 --noop_max 30 --frame_stack 10 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_frame_stack_uniform_u2_6_fs10_seed14", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 14, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-fs10-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "frame_stack", + "uniform", + "uniform_u2_6", + "fs10", + "seed14" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 15.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 10, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 14, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs10_seed14/episode_metrics.jsonl" + }, + "git_hash": "eb3a2e1efbd2aa03a60d7f44f5e18d8fdd0f5a2d", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/context_window/train/frame_stack/uniform_u2_6/fs10/seed_14", + "wandb_unique_id": "demon-attack-fs10-uniform_u2_6-s14" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs11:obs15:stride4:seed10/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs11:obs15:stride4:seed10/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..c38e5bceff5b5ba04e1f91fe768f0b445f01a9fd --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs11:obs15:stride4:seed10/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c7ea434aa8641a99366952230b35aac7cacf187d9a1ca0a01d0ad8685760ff14 +size 8058429 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs11:obs15:stride4:seed10/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs11:obs15:stride4:seed10/config.json new file mode 100644 index 0000000000000000000000000000000000000000..67a66af6446e7884266206881790ecf92b194d0e --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs11:obs15:stride4:seed10/config.json @@ -0,0 +1,255 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_frame_stack_uniform_u2_6_fs11_seed10", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 10, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-fs11-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "frame_stack", + "uniform", + "uniform_u2_6", + "fs11", + "seed10" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 15.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 11, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 10, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs11_seed10/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_frame_stack_uniform_u2_6_fs11_seed10 --train_dir results/checkpoints_factor_sweeps/demon_attack/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 10 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs11_seed10/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 10 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 36000 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-fs11-uniform_u2_6 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack frame_stack uniform uniform_u2_6 fs11 seed10 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 15.0 --screen_size 84 --noop_max 30 --frame_stack 11 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_frame_stack_uniform_u2_6_fs11_seed10", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 10, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-fs11-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "frame_stack", + "uniform", + "uniform_u2_6", + "fs11", + "seed10" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 15.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 11, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 10, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs11_seed10/episode_metrics.jsonl" + }, + "git_hash": "eb3a2e1efbd2aa03a60d7f44f5e18d8fdd0f5a2d", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/context_window/train/frame_stack/uniform_u2_6/fs11/seed_10", + "wandb_unique_id": "demon-attack-fs11-uniform_u2_6-s10" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs11:obs15:stride4:seed11/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs11:obs15:stride4:seed11/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..4a1fd477ce5e90adf9050b2a514c3f48dc870315 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs11:obs15:stride4:seed11/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:31cf534081733e58389ecfab7c9d6d1d3e9958a14bcf68018b0ae2681cf4333e +size 8058429 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs11:obs15:stride4:seed11/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs11:obs15:stride4:seed11/config.json new file mode 100644 index 0000000000000000000000000000000000000000..2c13afb97191f89045fc907776b2a01a9b6fa3cb --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs11:obs15:stride4:seed11/config.json @@ -0,0 +1,255 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_frame_stack_uniform_u2_6_fs11_seed11", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 11, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-fs11-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "frame_stack", + "uniform", + "uniform_u2_6", + "fs11", + "seed11" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 15.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 11, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 11, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs11_seed11/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_frame_stack_uniform_u2_6_fs11_seed11 --train_dir results/checkpoints_factor_sweeps/demon_attack/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 11 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs11_seed11/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 11 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 36000 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-fs11-uniform_u2_6 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack frame_stack uniform uniform_u2_6 fs11 seed11 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 15.0 --screen_size 84 --noop_max 30 --frame_stack 11 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_frame_stack_uniform_u2_6_fs11_seed11", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 11, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-fs11-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "frame_stack", + "uniform", + "uniform_u2_6", + "fs11", + "seed11" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 15.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 11, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 11, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs11_seed11/episode_metrics.jsonl" + }, + "git_hash": "eb3a2e1efbd2aa03a60d7f44f5e18d8fdd0f5a2d", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/context_window/train/frame_stack/uniform_u2_6/fs11/seed_11", + "wandb_unique_id": "demon-attack-fs11-uniform_u2_6-s11" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs11:obs15:stride4:seed12/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs11:obs15:stride4:seed12/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..307580752c1f93971a3cf6a1c9b7ddf574b45aeb --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs11:obs15:stride4:seed12/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ed5f20466a803db45b8efbc4b5774f7b9897a5859fa07dabb1e632afc0931ffe +size 8058429 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs11:obs15:stride4:seed12/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs11:obs15:stride4:seed12/config.json new file mode 100644 index 0000000000000000000000000000000000000000..e54bf66a616b8eb473c4d084662c16e0cc071ecb --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs11:obs15:stride4:seed12/config.json @@ -0,0 +1,255 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_frame_stack_uniform_u2_6_fs11_seed12", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 12, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-fs11-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "frame_stack", + "uniform", + "uniform_u2_6", + "fs11", + "seed12" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 15.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 11, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 12, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs11_seed12/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_frame_stack_uniform_u2_6_fs11_seed12 --train_dir results/checkpoints_factor_sweeps/demon_attack/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 12 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs11_seed12/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 12 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 36000 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-fs11-uniform_u2_6 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack frame_stack uniform uniform_u2_6 fs11 seed12 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 15.0 --screen_size 84 --noop_max 30 --frame_stack 11 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_frame_stack_uniform_u2_6_fs11_seed12", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 12, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-fs11-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "frame_stack", + "uniform", + "uniform_u2_6", + "fs11", + "seed12" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 15.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 11, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 12, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs11_seed12/episode_metrics.jsonl" + }, + "git_hash": "eb3a2e1efbd2aa03a60d7f44f5e18d8fdd0f5a2d", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/context_window/train/frame_stack/uniform_u2_6/fs11/seed_12", + "wandb_unique_id": "demon-attack-fs11-uniform_u2_6-s12" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs11:obs15:stride4:seed13/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs11:obs15:stride4:seed13/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..c39c8881ba33783a12c584edaa90e7281e8eb353 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs11:obs15:stride4:seed13/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e37c8d58bf41b2143f2f66a4cd7f4af5fd71fdb4fb78990c27150b2d68af315b +size 8058429 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs11:obs15:stride4:seed13/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs11:obs15:stride4:seed13/config.json new file mode 100644 index 0000000000000000000000000000000000000000..f4333d07516a3e5046d2fd77a09003926d1430d1 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs11:obs15:stride4:seed13/config.json @@ -0,0 +1,255 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_frame_stack_uniform_u2_6_fs11_seed13", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 13, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-fs11-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "frame_stack", + "uniform", + "uniform_u2_6", + "fs11", + "seed13" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 15.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 11, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 13, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs11_seed13/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_frame_stack_uniform_u2_6_fs11_seed13 --train_dir results/checkpoints_factor_sweeps/demon_attack/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 13 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs11_seed13/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 13 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 36000 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-fs11-uniform_u2_6 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack frame_stack uniform uniform_u2_6 fs11 seed13 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 15.0 --screen_size 84 --noop_max 30 --frame_stack 11 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_frame_stack_uniform_u2_6_fs11_seed13", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 13, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-fs11-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "frame_stack", + "uniform", + "uniform_u2_6", + "fs11", + "seed13" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 15.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 11, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 13, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs11_seed13/episode_metrics.jsonl" + }, + "git_hash": "eb3a2e1efbd2aa03a60d7f44f5e18d8fdd0f5a2d", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/context_window/train/frame_stack/uniform_u2_6/fs11/seed_13", + "wandb_unique_id": "demon-attack-fs11-uniform_u2_6-s13" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs11:obs15:stride4:seed14/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs11:obs15:stride4:seed14/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..237ac3e9d515aba82169460224599b57aa8e1d40 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs11:obs15:stride4:seed14/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b643c5cfee6c062ace24c8fc4904d330ce958b659005c6125a565ec5d8d55291 +size 8058429 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs11:obs15:stride4:seed14/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs11:obs15:stride4:seed14/config.json new file mode 100644 index 0000000000000000000000000000000000000000..4ba44aceba82e59dec39e766cbf3255290da28f5 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs11:obs15:stride4:seed14/config.json @@ -0,0 +1,255 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_frame_stack_uniform_u2_6_fs11_seed14", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 14, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-fs11-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "frame_stack", + "uniform", + "uniform_u2_6", + "fs11", + "seed14" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 15.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 11, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 14, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs11_seed14/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_frame_stack_uniform_u2_6_fs11_seed14 --train_dir results/checkpoints_factor_sweeps/demon_attack/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 14 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs11_seed14/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 14 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 36000 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-fs11-uniform_u2_6 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack frame_stack uniform uniform_u2_6 fs11 seed14 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 15.0 --screen_size 84 --noop_max 30 --frame_stack 11 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_frame_stack_uniform_u2_6_fs11_seed14", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 14, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-fs11-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "frame_stack", + "uniform", + "uniform_u2_6", + "fs11", + "seed14" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 15.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 11, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 14, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs11_seed14/episode_metrics.jsonl" + }, + "git_hash": "eb3a2e1efbd2aa03a60d7f44f5e18d8fdd0f5a2d", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/context_window/train/frame_stack/uniform_u2_6/fs11/seed_14", + "wandb_unique_id": "demon-attack-fs11-uniform_u2_6-s14" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs12:obs15:stride4:seed10/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs12:obs15:stride4:seed10/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..af3db7558c491babdf4b6a24c1dcf1e066820eeb --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs12:obs15:stride4:seed10/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1b3a4e32d55a89eccfe74620006f04f5b8419640d1b6b0ec43a513af9042b906 +size 8179517 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs12:obs15:stride4:seed10/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs12:obs15:stride4:seed10/config.json new file mode 100644 index 0000000000000000000000000000000000000000..bd66c7f787b139be11281dd92a1cb39cd487ecde --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs12:obs15:stride4:seed10/config.json @@ -0,0 +1,255 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_frame_stack_uniform_u2_6_fs12_seed10", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 10, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-fs12-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "frame_stack", + "uniform", + "uniform_u2_6", + "fs12", + "seed10" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 15.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 12, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 10, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs12_seed10/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_frame_stack_uniform_u2_6_fs12_seed10 --train_dir results/checkpoints_factor_sweeps/demon_attack/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 10 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs12_seed10/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 10 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 36000 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-fs12-uniform_u2_6 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack frame_stack uniform uniform_u2_6 fs12 seed10 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 15.0 --screen_size 84 --noop_max 30 --frame_stack 12 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_frame_stack_uniform_u2_6_fs12_seed10", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 10, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-fs12-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "frame_stack", + "uniform", + "uniform_u2_6", + "fs12", + "seed10" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 15.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 12, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 10, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs12_seed10/episode_metrics.jsonl" + }, + "git_hash": "eb3a2e1efbd2aa03a60d7f44f5e18d8fdd0f5a2d", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/context_window/train/frame_stack/uniform_u2_6/fs12/seed_10", + "wandb_unique_id": "demon-attack-fs12-uniform_u2_6-s10" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs12:obs15:stride4:seed11/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs12:obs15:stride4:seed11/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..e05ca10be9e8484afca0ab608b9a8c46839e510f --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs12:obs15:stride4:seed11/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:77fae23d456b496f72c322d935af05d0c86614fa393c94d40063fc15b28aa70b +size 8179517 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs12:obs15:stride4:seed11/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs12:obs15:stride4:seed11/config.json new file mode 100644 index 0000000000000000000000000000000000000000..c68e9885ec34cadeaf5601ec9fdd49bf341b29fd --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs12:obs15:stride4:seed11/config.json @@ -0,0 +1,255 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_frame_stack_uniform_u2_6_fs12_seed11", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 11, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-fs12-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "frame_stack", + "uniform", + "uniform_u2_6", + "fs12", + "seed11" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 15.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 12, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 11, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs12_seed11/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_frame_stack_uniform_u2_6_fs12_seed11 --train_dir results/checkpoints_factor_sweeps/demon_attack/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 11 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs12_seed11/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 11 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 36000 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-fs12-uniform_u2_6 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack frame_stack uniform uniform_u2_6 fs12 seed11 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 15.0 --screen_size 84 --noop_max 30 --frame_stack 12 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_frame_stack_uniform_u2_6_fs12_seed11", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 11, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-fs12-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "frame_stack", + "uniform", + "uniform_u2_6", + "fs12", + "seed11" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 15.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 12, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 11, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs12_seed11/episode_metrics.jsonl" + }, + "git_hash": "eb3a2e1efbd2aa03a60d7f44f5e18d8fdd0f5a2d", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/context_window/train/frame_stack/uniform_u2_6/fs12/seed_11", + "wandb_unique_id": "demon-attack-fs12-uniform_u2_6-s11" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs12:obs15:stride4:seed12/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs12:obs15:stride4:seed12/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..5672c44ffca4940d3c797be67017092526cad176 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs12:obs15:stride4:seed12/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:27554d3be4fca11bd98ab6e9ebceea30222c8fff55ce68aae64d2a41c00a9d29 +size 8179517 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs12:obs15:stride4:seed12/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs12:obs15:stride4:seed12/config.json new file mode 100644 index 0000000000000000000000000000000000000000..3b02d05e1ace72525720f209d8ee42952169f898 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs12:obs15:stride4:seed12/config.json @@ -0,0 +1,255 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_frame_stack_uniform_u2_6_fs12_seed12", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 12, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-fs12-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "frame_stack", + "uniform", + "uniform_u2_6", + "fs12", + "seed12" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 15.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 12, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 12, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs12_seed12/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_frame_stack_uniform_u2_6_fs12_seed12 --train_dir results/checkpoints_factor_sweeps/demon_attack/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 12 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs12_seed12/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 12 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 36000 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-fs12-uniform_u2_6 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack frame_stack uniform uniform_u2_6 fs12 seed12 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 15.0 --screen_size 84 --noop_max 30 --frame_stack 12 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_frame_stack_uniform_u2_6_fs12_seed12", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 12, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-fs12-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "frame_stack", + "uniform", + "uniform_u2_6", + "fs12", + "seed12" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 15.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 12, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 12, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs12_seed12/episode_metrics.jsonl" + }, + "git_hash": "eb3a2e1efbd2aa03a60d7f44f5e18d8fdd0f5a2d", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/context_window/train/frame_stack/uniform_u2_6/fs12/seed_12", + "wandb_unique_id": "demon-attack-fs12-uniform_u2_6-s12" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs12:obs15:stride4:seed13/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs12:obs15:stride4:seed13/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..967bb34617b23bb87814dc7fafce43e1b72a1ec3 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs12:obs15:stride4:seed13/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3b10e00b1a92f6d81791d76e4781830348fde29c7a863f7f0516cd94a27b2c2e +size 8179517 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs12:obs15:stride4:seed13/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs12:obs15:stride4:seed13/config.json new file mode 100644 index 0000000000000000000000000000000000000000..0f0b31713d5236ed1d2043653d0951e321901754 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs12:obs15:stride4:seed13/config.json @@ -0,0 +1,255 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_frame_stack_uniform_u2_6_fs12_seed13", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 13, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-fs12-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "frame_stack", + "uniform", + "uniform_u2_6", + "fs12", + "seed13" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 15.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 12, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 13, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs12_seed13/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_frame_stack_uniform_u2_6_fs12_seed13 --train_dir results/checkpoints_factor_sweeps/demon_attack/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 13 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs12_seed13/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 13 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 36000 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-fs12-uniform_u2_6 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack frame_stack uniform uniform_u2_6 fs12 seed13 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 15.0 --screen_size 84 --noop_max 30 --frame_stack 12 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_frame_stack_uniform_u2_6_fs12_seed13", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 13, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-fs12-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "frame_stack", + "uniform", + "uniform_u2_6", + "fs12", + "seed13" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 15.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 12, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 13, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs12_seed13/episode_metrics.jsonl" + }, + "git_hash": "eb3a2e1efbd2aa03a60d7f44f5e18d8fdd0f5a2d", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/context_window/train/frame_stack/uniform_u2_6/fs12/seed_13", + "wandb_unique_id": "demon-attack-fs12-uniform_u2_6-s13" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs4:obs15:stride4:seed10/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs4:obs15:stride4:seed10/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..2c95c6f5d22e5c7fe7ffc62474668555c4a30b2b --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs4:obs15:stride4:seed10/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:89d652bc51e3ac31ddf21294c5b9b5222e27b3c5de40529cd6afaa575b910b2f +size 7210813 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs4:obs15:stride4:seed10/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs4:obs15:stride4:seed10/config.json new file mode 100644 index 0000000000000000000000000000000000000000..2a448672021571bf362ee850e48006a09e561c12 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs4:obs15:stride4:seed10/config.json @@ -0,0 +1,255 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_frame_stack_uniform_u2_6_fs4_seed10", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 10, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-fs4-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "frame_stack", + "uniform", + "uniform_u2_6", + "fs4", + "seed10" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 15.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 10, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs4_seed10/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_frame_stack_uniform_u2_6_fs4_seed10 --train_dir results/checkpoints_factor_sweeps/demon_attack/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 10 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs4_seed10/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 10 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-fs4-uniform_u2_6 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack frame_stack uniform uniform_u2_6 fs4 seed10 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 15.0 --screen_size 84 --noop_max 30 --frame_stack 4 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_frame_stack_uniform_u2_6_fs4_seed10", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 10, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-fs4-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "frame_stack", + "uniform", + "uniform_u2_6", + "fs4", + "seed10" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 15.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 10, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs4_seed10/episode_metrics.jsonl" + }, + "git_hash": "a81783762c1f965cc4a9d2eb7f028b47ab17a859", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/context_window/train/frame_stack/uniform_u2_6/fs4/seed_10", + "wandb_unique_id": "demon-attack-fs4-uniform_u2_6-s10" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs4:obs15:stride4:seed11/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs4:obs15:stride4:seed11/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..3994e6824ba134f15ea1e39a4405328102f73d9d --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs4:obs15:stride4:seed11/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f185ff986491332f17c8f620166d706f12fd46519ec0b4ff410b84cb055f07a9 +size 7210813 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs4:obs15:stride4:seed11/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs4:obs15:stride4:seed11/config.json new file mode 100644 index 0000000000000000000000000000000000000000..7533e604572d28696aa474599fc3748e3f3eae72 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs4:obs15:stride4:seed11/config.json @@ -0,0 +1,255 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_frame_stack_uniform_u2_6_fs4_seed11", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 11, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-fs4-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "frame_stack", + "uniform", + "uniform_u2_6", + "fs4", + "seed11" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 15.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 11, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs4_seed11/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_frame_stack_uniform_u2_6_fs4_seed11 --train_dir results/checkpoints_factor_sweeps/demon_attack/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 11 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs4_seed11/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 11 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-fs4-uniform_u2_6 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack frame_stack uniform uniform_u2_6 fs4 seed11 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 15.0 --screen_size 84 --noop_max 30 --frame_stack 4 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_frame_stack_uniform_u2_6_fs4_seed11", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 11, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-fs4-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "frame_stack", + "uniform", + "uniform_u2_6", + "fs4", + "seed11" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 15.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 11, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs4_seed11/episode_metrics.jsonl" + }, + "git_hash": "a81783762c1f965cc4a9d2eb7f028b47ab17a859", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/context_window/train/frame_stack/uniform_u2_6/fs4/seed_11", + "wandb_unique_id": "demon-attack-fs4-uniform_u2_6-s11" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs4:obs15:stride4:seed12/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs4:obs15:stride4:seed12/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..4d9efb1c045a931070f8b95f752689bcb096cb6e --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs4:obs15:stride4:seed12/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1d69759fbea359b854109721ba87ae3f5e7812fff3c2371c7f92f495caa30a2e +size 7210813 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs4:obs15:stride4:seed12/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs4:obs15:stride4:seed12/config.json new file mode 100644 index 0000000000000000000000000000000000000000..b476d670453c268488d25dc89337c7d3820e170d --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs4:obs15:stride4:seed12/config.json @@ -0,0 +1,255 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_frame_stack_uniform_u2_6_fs4_seed12", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 12, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-fs4-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "frame_stack", + "uniform", + "uniform_u2_6", + "fs4", + "seed12" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 15.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 12, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs4_seed12/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_frame_stack_uniform_u2_6_fs4_seed12 --train_dir results/checkpoints_factor_sweeps/demon_attack/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 12 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs4_seed12/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 12 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-fs4-uniform_u2_6 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack frame_stack uniform uniform_u2_6 fs4 seed12 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 15.0 --screen_size 84 --noop_max 30 --frame_stack 4 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_frame_stack_uniform_u2_6_fs4_seed12", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 12, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-fs4-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "frame_stack", + "uniform", + "uniform_u2_6", + "fs4", + "seed12" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 15.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 12, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs4_seed12/episode_metrics.jsonl" + }, + "git_hash": "a81783762c1f965cc4a9d2eb7f028b47ab17a859", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/context_window/train/frame_stack/uniform_u2_6/fs4/seed_12", + "wandb_unique_id": "demon-attack-fs4-uniform_u2_6-s12" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs4:obs15:stride4:seed13/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs4:obs15:stride4:seed13/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..d63ef4121ed6db90f62af18d0fa5d8f3ed1f919b --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs4:obs15:stride4:seed13/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:213f4e418d3c60d62e9fbf67094607d43e53e5af93090ecb3bdefe984ce1b3da +size 7210813 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs4:obs15:stride4:seed13/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs4:obs15:stride4:seed13/config.json new file mode 100644 index 0000000000000000000000000000000000000000..44a4c4b6b93b215d618fd813e02f4825460320ed --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs4:obs15:stride4:seed13/config.json @@ -0,0 +1,255 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_frame_stack_uniform_u2_6_fs4_seed13", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 13, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-fs4-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "frame_stack", + "uniform", + "uniform_u2_6", + "fs4", + "seed13" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 15.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 13, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs4_seed13/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_frame_stack_uniform_u2_6_fs4_seed13 --train_dir results/checkpoints_factor_sweeps/demon_attack/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 13 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs4_seed13/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 13 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-fs4-uniform_u2_6 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack frame_stack uniform uniform_u2_6 fs4 seed13 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 15.0 --screen_size 84 --noop_max 30 --frame_stack 4 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_frame_stack_uniform_u2_6_fs4_seed13", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 13, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-fs4-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "frame_stack", + "uniform", + "uniform_u2_6", + "fs4", + "seed13" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 15.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 13, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs4_seed13/episode_metrics.jsonl" + }, + "git_hash": "a81783762c1f965cc4a9d2eb7f028b47ab17a859", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/context_window/train/frame_stack/uniform_u2_6/fs4/seed_13", + "wandb_unique_id": "demon-attack-fs4-uniform_u2_6-s13" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs4:obs15:stride4:seed14/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs4:obs15:stride4:seed14/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..de6299f3c041be975a7732ddf4f40904fdfc2e1f --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs4:obs15:stride4:seed14/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3052199dadf157e45685e7d170eb8cd2b0de5792a80a62bc4228372053d82e7e +size 7210813 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs4:obs15:stride4:seed14/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs4:obs15:stride4:seed14/config.json new file mode 100644 index 0000000000000000000000000000000000000000..8dd65f6c51c7cdd6ead3ce476b8856e2a65ff0a4 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs4:obs15:stride4:seed14/config.json @@ -0,0 +1,255 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_frame_stack_uniform_u2_6_fs4_seed14", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 14, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-fs4-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "frame_stack", + "uniform", + "uniform_u2_6", + "fs4", + "seed14" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 15.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 14, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs4_seed14/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_frame_stack_uniform_u2_6_fs4_seed14 --train_dir results/checkpoints_factor_sweeps/demon_attack/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 14 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs4_seed14/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 14 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-fs4-uniform_u2_6 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack frame_stack uniform uniform_u2_6 fs4 seed14 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 15.0 --screen_size 84 --noop_max 30 --frame_stack 4 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_frame_stack_uniform_u2_6_fs4_seed14", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 14, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-fs4-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "frame_stack", + "uniform", + "uniform_u2_6", + "fs4", + "seed14" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 15.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 14, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs4_seed14/episode_metrics.jsonl" + }, + "git_hash": "a81783762c1f965cc4a9d2eb7f028b47ab17a859", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/context_window/train/frame_stack/uniform_u2_6/fs4/seed_14", + "wandb_unique_id": "demon-attack-fs4-uniform_u2_6-s14" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs5:obs15:stride4:seed10/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs5:obs15:stride4:seed10/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..cb37c80f2bc74627722b1a4a40e936d7f331638d --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs5:obs15:stride4:seed10/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:67c5cf14508833f5106331ebf2f2cbc54f802d17bb7903e2d865ddb23ac34b2a +size 7331901 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs5:obs15:stride4:seed10/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs5:obs15:stride4:seed10/config.json new file mode 100644 index 0000000000000000000000000000000000000000..8febc79b8da99cd3554333f4eda575c988a44791 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs5:obs15:stride4:seed10/config.json @@ -0,0 +1,255 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_frame_stack_uniform_u2_6_fs5_seed10", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 10, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-fs5-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "frame_stack", + "uniform", + "uniform_u2_6", + "fs5", + "seed10" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 15.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 5, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 10, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs5_seed10/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_frame_stack_uniform_u2_6_fs5_seed10 --train_dir results/checkpoints_factor_sweeps/demon_attack/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 10 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs5_seed10/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 10 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-fs5-uniform_u2_6 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack frame_stack uniform uniform_u2_6 fs5 seed10 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 15.0 --screen_size 84 --noop_max 30 --frame_stack 5 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_frame_stack_uniform_u2_6_fs5_seed10", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 10, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-fs5-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "frame_stack", + "uniform", + "uniform_u2_6", + "fs5", + "seed10" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 15.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 5, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 10, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs5_seed10/episode_metrics.jsonl" + }, + "git_hash": "a81783762c1f965cc4a9d2eb7f028b47ab17a859", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/context_window/train/frame_stack/uniform_u2_6/fs5/seed_10", + "wandb_unique_id": "demon-attack-fs5-uniform_u2_6-s10" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs5:obs15:stride4:seed11/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs5:obs15:stride4:seed11/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..e91321c9556cb2bddd48036cb448c10504b78f0d --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs5:obs15:stride4:seed11/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:47597f966b12da8f4e8720db6045f5ec1716672e93667400d7a508f39fcc9c2e +size 7331901 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs5:obs15:stride4:seed11/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs5:obs15:stride4:seed11/config.json new file mode 100644 index 0000000000000000000000000000000000000000..78a388652c410cf61f317f236f8b5e207c256205 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs5:obs15:stride4:seed11/config.json @@ -0,0 +1,255 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_frame_stack_uniform_u2_6_fs5_seed11", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 11, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-fs5-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "frame_stack", + "uniform", + "uniform_u2_6", + "fs5", + "seed11" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 15.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 5, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 11, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs5_seed11/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_frame_stack_uniform_u2_6_fs5_seed11 --train_dir results/checkpoints_factor_sweeps/demon_attack/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 11 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs5_seed11/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 11 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-fs5-uniform_u2_6 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack frame_stack uniform uniform_u2_6 fs5 seed11 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 15.0 --screen_size 84 --noop_max 30 --frame_stack 5 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_frame_stack_uniform_u2_6_fs5_seed11", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 11, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-fs5-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "frame_stack", + "uniform", + "uniform_u2_6", + "fs5", + "seed11" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 15.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 5, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 11, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs5_seed11/episode_metrics.jsonl" + }, + "git_hash": "a81783762c1f965cc4a9d2eb7f028b47ab17a859", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/context_window/train/frame_stack/uniform_u2_6/fs5/seed_11", + "wandb_unique_id": "demon-attack-fs5-uniform_u2_6-s11" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs5:obs15:stride4:seed12/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs5:obs15:stride4:seed12/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..ce45e93fa82047779556d5944ec0e5a0a5172bac --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs5:obs15:stride4:seed12/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f6041f7be23583fc1d6a37d253b5c45de1e2008f4ee88a6aa1271a243ca90859 +size 7331901 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs5:obs15:stride4:seed12/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs5:obs15:stride4:seed12/config.json new file mode 100644 index 0000000000000000000000000000000000000000..52fdc29b1f21e6c020b2e064c10820aa5b6672b6 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs5:obs15:stride4:seed12/config.json @@ -0,0 +1,255 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_frame_stack_uniform_u2_6_fs5_seed12", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 12, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-fs5-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "frame_stack", + "uniform", + "uniform_u2_6", + "fs5", + "seed12" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 15.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 5, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 12, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs5_seed12/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_frame_stack_uniform_u2_6_fs5_seed12 --train_dir results/checkpoints_factor_sweeps/demon_attack/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 12 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs5_seed12/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 12 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-fs5-uniform_u2_6 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack frame_stack uniform uniform_u2_6 fs5 seed12 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 15.0 --screen_size 84 --noop_max 30 --frame_stack 5 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_frame_stack_uniform_u2_6_fs5_seed12", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 12, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-fs5-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "frame_stack", + "uniform", + "uniform_u2_6", + "fs5", + "seed12" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 15.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 5, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 12, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs5_seed12/episode_metrics.jsonl" + }, + "git_hash": "a81783762c1f965cc4a9d2eb7f028b47ab17a859", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/context_window/train/frame_stack/uniform_u2_6/fs5/seed_12", + "wandb_unique_id": "demon-attack-fs5-uniform_u2_6-s12" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs5:obs15:stride4:seed13/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs5:obs15:stride4:seed13/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..1ec2ced6e1b65e1c013172315094935e7ab21275 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs5:obs15:stride4:seed13/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:56d7c5bf2bf28d163a2f7fb20f34c42d3caedcebd3e0e604821742eadafcf606 +size 7331901 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs5:obs15:stride4:seed13/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs5:obs15:stride4:seed13/config.json new file mode 100644 index 0000000000000000000000000000000000000000..08daea8fc4b8a21c3ed2c5030405e8dafb52caaa --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs5:obs15:stride4:seed13/config.json @@ -0,0 +1,255 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_frame_stack_uniform_u2_6_fs5_seed13", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 13, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-fs5-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "frame_stack", + "uniform", + "uniform_u2_6", + "fs5", + "seed13" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 15.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 5, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 13, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs5_seed13/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_frame_stack_uniform_u2_6_fs5_seed13 --train_dir results/checkpoints_factor_sweeps/demon_attack/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 13 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs5_seed13/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 13 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-fs5-uniform_u2_6 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack frame_stack uniform uniform_u2_6 fs5 seed13 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 15.0 --screen_size 84 --noop_max 30 --frame_stack 5 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_frame_stack_uniform_u2_6_fs5_seed13", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 13, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-fs5-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "frame_stack", + "uniform", + "uniform_u2_6", + "fs5", + "seed13" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 15.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 5, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 13, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs5_seed13/episode_metrics.jsonl" + }, + "git_hash": "a81783762c1f965cc4a9d2eb7f028b47ab17a859", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/context_window/train/frame_stack/uniform_u2_6/fs5/seed_13", + "wandb_unique_id": "demon-attack-fs5-uniform_u2_6-s13" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs5:obs15:stride4:seed14/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs5:obs15:stride4:seed14/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..d9703eebf931fcdd833b48d5bd50a24509ce39b6 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs5:obs15:stride4:seed14/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8ff7f9b130d633f55a642ede33485441297ea6b7b763546d0f3edd89a2d409dd +size 7331901 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs5:obs15:stride4:seed14/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs5:obs15:stride4:seed14/config.json new file mode 100644 index 0000000000000000000000000000000000000000..29242d8a3b27ef8bb639f6002005474e5f2a4b22 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs5:obs15:stride4:seed14/config.json @@ -0,0 +1,255 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_frame_stack_uniform_u2_6_fs5_seed14", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 14, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-fs5-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "frame_stack", + "uniform", + "uniform_u2_6", + "fs5", + "seed14" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 15.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 5, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 14, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs5_seed14/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_frame_stack_uniform_u2_6_fs5_seed14 --train_dir results/checkpoints_factor_sweeps/demon_attack/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 14 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs5_seed14/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 14 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-fs5-uniform_u2_6 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack frame_stack uniform uniform_u2_6 fs5 seed14 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 15.0 --screen_size 84 --noop_max 30 --frame_stack 5 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_frame_stack_uniform_u2_6_fs5_seed14", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 14, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-fs5-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "frame_stack", + "uniform", + "uniform_u2_6", + "fs5", + "seed14" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 15.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 5, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 14, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs5_seed14/episode_metrics.jsonl" + }, + "git_hash": "a81783762c1f965cc4a9d2eb7f028b47ab17a859", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/context_window/train/frame_stack/uniform_u2_6/fs5/seed_14", + "wandb_unique_id": "demon-attack-fs5-uniform_u2_6-s14" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs6:obs15:stride4:seed10/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs6:obs15:stride4:seed10/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..ba3a3b1db7eb00ea4cb1f440547f9e46d196519c --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs6:obs15:stride4:seed10/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6b317e768314d95b0e485def34bee8e44c385a46e4d862d206741e3f1b8a80be +size 7452989 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs6:obs15:stride4:seed10/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs6:obs15:stride4:seed10/config.json new file mode 100644 index 0000000000000000000000000000000000000000..a916c8e08c31d1ac86c7fb387f516c41af79c649 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs6:obs15:stride4:seed10/config.json @@ -0,0 +1,255 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_frame_stack_uniform_u2_6_fs6_seed10", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 10, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-fs6-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "frame_stack", + "uniform", + "uniform_u2_6", + "fs6", + "seed10" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 15.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 6, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 10, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs6_seed10/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_frame_stack_uniform_u2_6_fs6_seed10 --train_dir results/checkpoints_factor_sweeps/demon_attack/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 10 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs6_seed10/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 10 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-fs6-uniform_u2_6 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack frame_stack uniform uniform_u2_6 fs6 seed10 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 15.0 --screen_size 84 --noop_max 30 --frame_stack 6 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_frame_stack_uniform_u2_6_fs6_seed10", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 10, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-fs6-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "frame_stack", + "uniform", + "uniform_u2_6", + "fs6", + "seed10" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 15.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 6, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 10, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs6_seed10/episode_metrics.jsonl" + }, + "git_hash": "a81783762c1f965cc4a9d2eb7f028b47ab17a859", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/context_window/train/frame_stack/uniform_u2_6/fs6/seed_10", + "wandb_unique_id": "demon-attack-fs6-uniform_u2_6-s10" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs6:obs15:stride4:seed11/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs6:obs15:stride4:seed11/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..85e470e4b2c4ba82e79943ef16ace7353601d154 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs6:obs15:stride4:seed11/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5a2b5c018f7add7f106beb2941ecfe12f2bf0322f0aebb3b604ea300c87801f5 +size 7452989 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs6:obs15:stride4:seed11/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs6:obs15:stride4:seed11/config.json new file mode 100644 index 0000000000000000000000000000000000000000..395999504ef69472ddf789913374efd272c7a7be --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs6:obs15:stride4:seed11/config.json @@ -0,0 +1,255 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_frame_stack_uniform_u2_6_fs6_seed11", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 11, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-fs6-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "frame_stack", + "uniform", + "uniform_u2_6", + "fs6", + "seed11" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 15.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 6, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 11, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs6_seed11/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_frame_stack_uniform_u2_6_fs6_seed11 --train_dir results/checkpoints_factor_sweeps/demon_attack/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 11 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs6_seed11/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 11 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-fs6-uniform_u2_6 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack frame_stack uniform uniform_u2_6 fs6 seed11 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 15.0 --screen_size 84 --noop_max 30 --frame_stack 6 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_frame_stack_uniform_u2_6_fs6_seed11", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 11, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-fs6-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "frame_stack", + "uniform", + "uniform_u2_6", + "fs6", + "seed11" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 15.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 6, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 11, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs6_seed11/episode_metrics.jsonl" + }, + "git_hash": "a81783762c1f965cc4a9d2eb7f028b47ab17a859", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/context_window/train/frame_stack/uniform_u2_6/fs6/seed_11", + "wandb_unique_id": "demon-attack-fs6-uniform_u2_6-s11" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs6:obs15:stride4:seed12/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs6:obs15:stride4:seed12/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..6376a9f47108db151b5c9f1f598217beb43479cb --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs6:obs15:stride4:seed12/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:54e6e2fef3d79a2e37452787ed4e59645b75220b01b0f7c81c3a760d1e2dd29e +size 7452989 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs6:obs15:stride4:seed12/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs6:obs15:stride4:seed12/config.json new file mode 100644 index 0000000000000000000000000000000000000000..1099c67568ddf9a6cab3dd418d6912385b0b5051 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs6:obs15:stride4:seed12/config.json @@ -0,0 +1,255 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_frame_stack_uniform_u2_6_fs6_seed12", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 12, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-fs6-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "frame_stack", + "uniform", + "uniform_u2_6", + "fs6", + "seed12" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 15.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 6, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 12, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs6_seed12/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_frame_stack_uniform_u2_6_fs6_seed12 --train_dir results/checkpoints_factor_sweeps/demon_attack/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 12 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs6_seed12/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 12 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-fs6-uniform_u2_6 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack frame_stack uniform uniform_u2_6 fs6 seed12 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 15.0 --screen_size 84 --noop_max 30 --frame_stack 6 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_frame_stack_uniform_u2_6_fs6_seed12", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 12, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-fs6-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "frame_stack", + "uniform", + "uniform_u2_6", + "fs6", + "seed12" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 15.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 6, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 12, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs6_seed12/episode_metrics.jsonl" + }, + "git_hash": "a81783762c1f965cc4a9d2eb7f028b47ab17a859", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/context_window/train/frame_stack/uniform_u2_6/fs6/seed_12", + "wandb_unique_id": "demon-attack-fs6-uniform_u2_6-s12" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs6:obs15:stride4:seed13/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs6:obs15:stride4:seed13/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..b0ffd6be27854f009afec19ebcfd0238a3553926 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs6:obs15:stride4:seed13/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fdf1358dadff698a4fbd1d9bcc97470d3bc0b0eb46a7221c8c851d0a8dc6f8cd +size 7452989 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs6:obs15:stride4:seed13/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs6:obs15:stride4:seed13/config.json new file mode 100644 index 0000000000000000000000000000000000000000..f64e61e9e71a8dd28defd1f6d92977f1da180155 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs6:obs15:stride4:seed13/config.json @@ -0,0 +1,255 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_frame_stack_uniform_u2_6_fs6_seed13", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 13, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-fs6-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "frame_stack", + "uniform", + "uniform_u2_6", + "fs6", + "seed13" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 15.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 6, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 13, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs6_seed13/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_frame_stack_uniform_u2_6_fs6_seed13 --train_dir results/checkpoints_factor_sweeps/demon_attack/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 13 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs6_seed13/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 13 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-fs6-uniform_u2_6 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack frame_stack uniform uniform_u2_6 fs6 seed13 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 15.0 --screen_size 84 --noop_max 30 --frame_stack 6 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_frame_stack_uniform_u2_6_fs6_seed13", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 13, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-fs6-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "frame_stack", + "uniform", + "uniform_u2_6", + "fs6", + "seed13" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 15.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 6, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 13, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs6_seed13/episode_metrics.jsonl" + }, + "git_hash": "a81783762c1f965cc4a9d2eb7f028b47ab17a859", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/context_window/train/frame_stack/uniform_u2_6/fs6/seed_13", + "wandb_unique_id": "demon-attack-fs6-uniform_u2_6-s13" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs6:obs15:stride4:seed14/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs6:obs15:stride4:seed14/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..2450afa5659ce7be431528a537587ebcb7b3dd16 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs6:obs15:stride4:seed14/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:256205b9274992eb5efb3e0434fc4935bd81325e74d4811ed0243a8d4d7657a6 +size 7452989 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs6:obs15:stride4:seed14/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs6:obs15:stride4:seed14/config.json new file mode 100644 index 0000000000000000000000000000000000000000..90d7407a5e36ee04fb62c43681f5ee93f10975e1 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs6:obs15:stride4:seed14/config.json @@ -0,0 +1,255 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_frame_stack_uniform_u2_6_fs6_seed14", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 14, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-fs6-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "frame_stack", + "uniform", + "uniform_u2_6", + "fs6", + "seed14" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 15.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 6, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 14, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs6_seed14/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_frame_stack_uniform_u2_6_fs6_seed14 --train_dir results/checkpoints_factor_sweeps/demon_attack/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 14 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs6_seed14/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 14 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-fs6-uniform_u2_6 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack frame_stack uniform uniform_u2_6 fs6 seed14 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 15.0 --screen_size 84 --noop_max 30 --frame_stack 6 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_frame_stack_uniform_u2_6_fs6_seed14", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 14, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-fs6-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "frame_stack", + "uniform", + "uniform_u2_6", + "fs6", + "seed14" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 15.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 6, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 14, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs6_seed14/episode_metrics.jsonl" + }, + "git_hash": "a81783762c1f965cc4a9d2eb7f028b47ab17a859", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/context_window/train/frame_stack/uniform_u2_6/fs6/seed_14", + "wandb_unique_id": "demon-attack-fs6-uniform_u2_6-s14" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs7:obs15:stride4:seed10/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs7:obs15:stride4:seed10/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..4e82762c6efda63d23c45611a496a26fe0ce8647 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs7:obs15:stride4:seed10/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:721d1d5ef7209c3f08a806915049d36b383f7c2fbf253fe936cc6cb1ef7311f1 +size 7574077 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs7:obs15:stride4:seed10/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs7:obs15:stride4:seed10/config.json new file mode 100644 index 0000000000000000000000000000000000000000..0356f1a7a3f2f3aebba4d4e86f719778591d561a --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs7:obs15:stride4:seed10/config.json @@ -0,0 +1,255 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_frame_stack_uniform_u2_6_fs7_seed10", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 10, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-fs7-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "frame_stack", + "uniform", + "uniform_u2_6", + "fs7", + "seed10" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 15.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 7, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 10, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs7_seed10/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_frame_stack_uniform_u2_6_fs7_seed10 --train_dir results/checkpoints_factor_sweeps/demon_attack/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 10 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs7_seed10/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 10 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-fs7-uniform_u2_6 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack frame_stack uniform uniform_u2_6 fs7 seed10 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 15.0 --screen_size 84 --noop_max 30 --frame_stack 7 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_frame_stack_uniform_u2_6_fs7_seed10", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 10, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-fs7-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "frame_stack", + "uniform", + "uniform_u2_6", + "fs7", + "seed10" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 15.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 7, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 10, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs7_seed10/episode_metrics.jsonl" + }, + "git_hash": "a81783762c1f965cc4a9d2eb7f028b47ab17a859", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/context_window/train/frame_stack/uniform_u2_6/fs7/seed_10", + "wandb_unique_id": "demon-attack-fs7-uniform_u2_6-s10" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs7:obs15:stride4:seed11/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs7:obs15:stride4:seed11/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..02f959bc26eeab04bb5ae248ecfb72be8f8e729b --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs7:obs15:stride4:seed11/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ad4eba2c6d6b20ddb0d55f09f6a8599cff44cc2bbcfb4c96efec0c528ecd4a1b +size 7574077 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs7:obs15:stride4:seed11/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs7:obs15:stride4:seed11/config.json new file mode 100644 index 0000000000000000000000000000000000000000..de58404375f685febd5684331058f11025ded719 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs7:obs15:stride4:seed11/config.json @@ -0,0 +1,255 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_frame_stack_uniform_u2_6_fs7_seed11", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 11, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-fs7-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "frame_stack", + "uniform", + "uniform_u2_6", + "fs7", + "seed11" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 15.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 7, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 11, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs7_seed11/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_frame_stack_uniform_u2_6_fs7_seed11 --train_dir results/checkpoints_factor_sweeps/demon_attack/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 11 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs7_seed11/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 11 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-fs7-uniform_u2_6 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack frame_stack uniform uniform_u2_6 fs7 seed11 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 15.0 --screen_size 84 --noop_max 30 --frame_stack 7 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_frame_stack_uniform_u2_6_fs7_seed11", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 11, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-fs7-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "frame_stack", + "uniform", + "uniform_u2_6", + "fs7", + "seed11" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 15.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 7, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 11, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs7_seed11/episode_metrics.jsonl" + }, + "git_hash": "a81783762c1f965cc4a9d2eb7f028b47ab17a859", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/context_window/train/frame_stack/uniform_u2_6/fs7/seed_11", + "wandb_unique_id": "demon-attack-fs7-uniform_u2_6-s11" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs7:obs15:stride4:seed12/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs7:obs15:stride4:seed12/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..99367ac893d976720fb3228fa27f0c533d624bdc --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs7:obs15:stride4:seed12/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e240629f0f4e9ee8e1a45d900d81a9672c82444012f8afc4017a766b025dfce5 +size 7574077 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs7:obs15:stride4:seed12/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs7:obs15:stride4:seed12/config.json new file mode 100644 index 0000000000000000000000000000000000000000..29702ac8a37132b664942f9c4699993629118035 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs7:obs15:stride4:seed12/config.json @@ -0,0 +1,255 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_frame_stack_uniform_u2_6_fs7_seed12", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 12, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-fs7-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "frame_stack", + "uniform", + "uniform_u2_6", + "fs7", + "seed12" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 15.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 7, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 12, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs7_seed12/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_frame_stack_uniform_u2_6_fs7_seed12 --train_dir results/checkpoints_factor_sweeps/demon_attack/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 12 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs7_seed12/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 12 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-fs7-uniform_u2_6 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack frame_stack uniform uniform_u2_6 fs7 seed12 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 15.0 --screen_size 84 --noop_max 30 --frame_stack 7 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_frame_stack_uniform_u2_6_fs7_seed12", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 12, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-fs7-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "frame_stack", + "uniform", + "uniform_u2_6", + "fs7", + "seed12" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 15.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 7, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 12, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs7_seed12/episode_metrics.jsonl" + }, + "git_hash": "a81783762c1f965cc4a9d2eb7f028b47ab17a859", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/context_window/train/frame_stack/uniform_u2_6/fs7/seed_12", + "wandb_unique_id": "demon-attack-fs7-uniform_u2_6-s12" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs7:obs15:stride4:seed13/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs7:obs15:stride4:seed13/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..f6efa0c012547c37dad2ef9ca7224bc19425d869 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs7:obs15:stride4:seed13/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5319fcee7bd67a0319e9df5dd72a1afc8d4de389e3f02b0c16e782af65f69adf +size 7574077 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs7:obs15:stride4:seed13/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs7:obs15:stride4:seed13/config.json new file mode 100644 index 0000000000000000000000000000000000000000..af2d561bb39d9053851a74ed4e32020c0ec277ce --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs7:obs15:stride4:seed13/config.json @@ -0,0 +1,255 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_frame_stack_uniform_u2_6_fs7_seed13", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 13, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-fs7-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "frame_stack", + "uniform", + "uniform_u2_6", + "fs7", + "seed13" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 15.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 7, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 13, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs7_seed13/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_frame_stack_uniform_u2_6_fs7_seed13 --train_dir results/checkpoints_factor_sweeps/demon_attack/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 13 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs7_seed13/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 13 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-fs7-uniform_u2_6 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack frame_stack uniform uniform_u2_6 fs7 seed13 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 15.0 --screen_size 84 --noop_max 30 --frame_stack 7 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_frame_stack_uniform_u2_6_fs7_seed13", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 13, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-fs7-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "frame_stack", + "uniform", + "uniform_u2_6", + "fs7", + "seed13" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 15.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 7, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 13, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs7_seed13/episode_metrics.jsonl" + }, + "git_hash": "a81783762c1f965cc4a9d2eb7f028b47ab17a859", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/context_window/train/frame_stack/uniform_u2_6/fs7/seed_13", + "wandb_unique_id": "demon-attack-fs7-uniform_u2_6-s13" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs7:obs15:stride4:seed14/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs7:obs15:stride4:seed14/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..7a7f8294e13c394ad82fb7588b51da75ed90701c --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs7:obs15:stride4:seed14/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80ffb7836ab01d74fb3481d3940ab600bb03a6f0c6a097fd156c253c29f897e9 +size 7574077 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs7:obs15:stride4:seed14/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs7:obs15:stride4:seed14/config.json new file mode 100644 index 0000000000000000000000000000000000000000..472712224b421c69e402cbf279c2c829d08fdf4a --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs7:obs15:stride4:seed14/config.json @@ -0,0 +1,255 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_frame_stack_uniform_u2_6_fs7_seed14", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 14, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-fs7-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "frame_stack", + "uniform", + "uniform_u2_6", + "fs7", + "seed14" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 15.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 7, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 14, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs7_seed14/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_frame_stack_uniform_u2_6_fs7_seed14 --train_dir results/checkpoints_factor_sweeps/demon_attack/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 14 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs7_seed14/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 14 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-fs7-uniform_u2_6 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack frame_stack uniform uniform_u2_6 fs7 seed14 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 15.0 --screen_size 84 --noop_max 30 --frame_stack 7 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_frame_stack_uniform_u2_6_fs7_seed14", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 14, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-fs7-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "frame_stack", + "uniform", + "uniform_u2_6", + "fs7", + "seed14" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 15.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 7, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 14, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs7_seed14/episode_metrics.jsonl" + }, + "git_hash": "a81783762c1f965cc4a9d2eb7f028b47ab17a859", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/context_window/train/frame_stack/uniform_u2_6/fs7/seed_14", + "wandb_unique_id": "demon-attack-fs7-uniform_u2_6-s14" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs8:obs15:stride4:seed10/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs8:obs15:stride4:seed10/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..15d98fdf266159adcc2eab730d267184d00c4f58 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs8:obs15:stride4:seed10/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1a747d768d8beeac3ff0f37f16bac86fc9019d729f5cf193b8bafaf5399eca00 +size 7695165 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs8:obs15:stride4:seed10/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs8:obs15:stride4:seed10/config.json new file mode 100644 index 0000000000000000000000000000000000000000..81c8d5e4e73f33a897d695557a93b49d8faa2b78 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs8:obs15:stride4:seed10/config.json @@ -0,0 +1,255 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_frame_stack_uniform_u2_6_fs8_seed10", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 10, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-fs8-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "frame_stack", + "uniform", + "uniform_u2_6", + "fs8", + "seed10" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 15.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 8, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 10, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs8_seed10/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_frame_stack_uniform_u2_6_fs8_seed10 --train_dir results/checkpoints_factor_sweeps/demon_attack/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 10 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs8_seed10/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 10 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-fs8-uniform_u2_6 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack frame_stack uniform uniform_u2_6 fs8 seed10 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 15.0 --screen_size 84 --noop_max 30 --frame_stack 8 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_frame_stack_uniform_u2_6_fs8_seed10", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 10, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-fs8-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "frame_stack", + "uniform", + "uniform_u2_6", + "fs8", + "seed10" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 15.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 8, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 10, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs8_seed10/episode_metrics.jsonl" + }, + "git_hash": "a81783762c1f965cc4a9d2eb7f028b47ab17a859", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/context_window/train/frame_stack/uniform_u2_6/fs8/seed_10", + "wandb_unique_id": "demon-attack-fs8-uniform_u2_6-s10" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs8:obs15:stride4:seed11/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs8:obs15:stride4:seed11/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..6b185192acd2a4d6abe98e8c61e41194baa46d00 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs8:obs15:stride4:seed11/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:54f6e6babed1b2d7d0ba825a497550b80304530a1508ec6140680d5a12d37538 +size 7695165 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs8:obs15:stride4:seed11/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs8:obs15:stride4:seed11/config.json new file mode 100644 index 0000000000000000000000000000000000000000..78dd41d02466acee8ceddff2d62a449317460d0d --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs8:obs15:stride4:seed11/config.json @@ -0,0 +1,255 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_frame_stack_uniform_u2_6_fs8_seed11", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 11, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-fs8-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "frame_stack", + "uniform", + "uniform_u2_6", + "fs8", + "seed11" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 15.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 8, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 11, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs8_seed11/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_frame_stack_uniform_u2_6_fs8_seed11 --train_dir results/checkpoints_factor_sweeps/demon_attack/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 11 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs8_seed11/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 11 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-fs8-uniform_u2_6 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack frame_stack uniform uniform_u2_6 fs8 seed11 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 15.0 --screen_size 84 --noop_max 30 --frame_stack 8 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_frame_stack_uniform_u2_6_fs8_seed11", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 11, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-fs8-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "frame_stack", + "uniform", + "uniform_u2_6", + "fs8", + "seed11" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 15.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 8, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 11, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs8_seed11/episode_metrics.jsonl" + }, + "git_hash": "a81783762c1f965cc4a9d2eb7f028b47ab17a859", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/context_window/train/frame_stack/uniform_u2_6/fs8/seed_11", + "wandb_unique_id": "demon-attack-fs8-uniform_u2_6-s11" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs8:obs15:stride4:seed12/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs8:obs15:stride4:seed12/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..c8e8b514b2a1ba3b29a953ce08be520516773742 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs8:obs15:stride4:seed12/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:564594eeb0db139c806c9d9a055eaae32edc124f38314d00d0c28f189af9807f +size 7695165 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs8:obs15:stride4:seed12/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs8:obs15:stride4:seed12/config.json new file mode 100644 index 0000000000000000000000000000000000000000..32387bd717558ae0248606e123e8073d97a212c7 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs8:obs15:stride4:seed12/config.json @@ -0,0 +1,255 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_frame_stack_uniform_u2_6_fs8_seed12", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 12, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-fs8-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "frame_stack", + "uniform", + "uniform_u2_6", + "fs8", + "seed12" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 15.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 8, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 12, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs8_seed12/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_frame_stack_uniform_u2_6_fs8_seed12 --train_dir results/checkpoints_factor_sweeps/demon_attack/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 12 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs8_seed12/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 12 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-fs8-uniform_u2_6 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack frame_stack uniform uniform_u2_6 fs8 seed12 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 15.0 --screen_size 84 --noop_max 30 --frame_stack 8 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_frame_stack_uniform_u2_6_fs8_seed12", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 12, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-fs8-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "frame_stack", + "uniform", + "uniform_u2_6", + "fs8", + "seed12" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 15.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 8, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 12, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs8_seed12/episode_metrics.jsonl" + }, + "git_hash": "a81783762c1f965cc4a9d2eb7f028b47ab17a859", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/context_window/train/frame_stack/uniform_u2_6/fs8/seed_12", + "wandb_unique_id": "demon-attack-fs8-uniform_u2_6-s12" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs8:obs15:stride4:seed13/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs8:obs15:stride4:seed13/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..05019f5a8584dcc8d7c3bc8f06e4c7f4aef0e995 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs8:obs15:stride4:seed13/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6455bb1ef5c1ee0b3c47ecd4c1f137f7d277dccf8d37feb63abd16b925f5b4c3 +size 7695165 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs8:obs15:stride4:seed13/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs8:obs15:stride4:seed13/config.json new file mode 100644 index 0000000000000000000000000000000000000000..aa72397e8477af9d5ea2f62becd22f74a678f3b1 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs8:obs15:stride4:seed13/config.json @@ -0,0 +1,255 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_frame_stack_uniform_u2_6_fs8_seed13", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 13, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-fs8-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "frame_stack", + "uniform", + "uniform_u2_6", + "fs8", + "seed13" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 15.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 8, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 13, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs8_seed13/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_frame_stack_uniform_u2_6_fs8_seed13 --train_dir results/checkpoints_factor_sweeps/demon_attack/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 13 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs8_seed13/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 13 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-fs8-uniform_u2_6 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack frame_stack uniform uniform_u2_6 fs8 seed13 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 15.0 --screen_size 84 --noop_max 30 --frame_stack 8 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_frame_stack_uniform_u2_6_fs8_seed13", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 13, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-fs8-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "frame_stack", + "uniform", + "uniform_u2_6", + "fs8", + "seed13" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 15.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 8, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 13, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs8_seed13/episode_metrics.jsonl" + }, + "git_hash": "a81783762c1f965cc4a9d2eb7f028b47ab17a859", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/context_window/train/frame_stack/uniform_u2_6/fs8/seed_13", + "wandb_unique_id": "demon-attack-fs8-uniform_u2_6-s13" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs8:obs15:stride4:seed14/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs8:obs15:stride4:seed14/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..e941d57180be3e0da0a63be22939bb2de76cb20b --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs8:obs15:stride4:seed14/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7c65f25a1bc4a02155b38d5d3e27cda8d15640f89287f382607709ea81e9fb6c +size 7695165 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs8:obs15:stride4:seed14/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs8:obs15:stride4:seed14/config.json new file mode 100644 index 0000000000000000000000000000000000000000..06aacca1c52a9552539bdaa97982730eaf0a47e8 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs8:obs15:stride4:seed14/config.json @@ -0,0 +1,255 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_frame_stack_uniform_u2_6_fs8_seed14", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 14, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-fs8-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "frame_stack", + "uniform", + "uniform_u2_6", + "fs8", + "seed14" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 15.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 8, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 14, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs8_seed14/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_frame_stack_uniform_u2_6_fs8_seed14 --train_dir results/checkpoints_factor_sweeps/demon_attack/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 14 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs8_seed14/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 14 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-fs8-uniform_u2_6 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack frame_stack uniform uniform_u2_6 fs8 seed14 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 15.0 --screen_size 84 --noop_max 30 --frame_stack 8 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_frame_stack_uniform_u2_6_fs8_seed14", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 14, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-fs8-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "frame_stack", + "uniform", + "uniform_u2_6", + "fs8", + "seed14" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 15.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 8, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 14, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs8_seed14/episode_metrics.jsonl" + }, + "git_hash": "a81783762c1f965cc4a9d2eb7f028b47ab17a859", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/context_window/train/frame_stack/uniform_u2_6/fs8/seed_14", + "wandb_unique_id": "demon-attack-fs8-uniform_u2_6-s14" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs9:obs15:stride4:seed10/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs9:obs15:stride4:seed10/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..2e3dcfc9857f9533d2c71e7cccd75d51cc6e31b5 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs9:obs15:stride4:seed10/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:12935a86dfba9e11a7a3cf834d5ec36f39fd087c9407f5d3c634bf0771e0c41c +size 7816253 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs9:obs15:stride4:seed10/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs9:obs15:stride4:seed10/config.json new file mode 100644 index 0000000000000000000000000000000000000000..c2ee18f9ed43772f3cfc7ccbba26b7cd864c8cfb --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs9:obs15:stride4:seed10/config.json @@ -0,0 +1,255 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_frame_stack_uniform_u2_6_fs9_seed10", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 10, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-fs9-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "frame_stack", + "uniform", + "uniform_u2_6", + "fs9", + "seed10" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 15.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 9, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 10, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs9_seed10/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_frame_stack_uniform_u2_6_fs9_seed10 --train_dir results/checkpoints_factor_sweeps/demon_attack/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 10 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs9_seed10/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 10 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 36000 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-fs9-uniform_u2_6 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack frame_stack uniform uniform_u2_6 fs9 seed10 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 15.0 --screen_size 84 --noop_max 30 --frame_stack 9 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_frame_stack_uniform_u2_6_fs9_seed10", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 10, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-fs9-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "frame_stack", + "uniform", + "uniform_u2_6", + "fs9", + "seed10" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 15.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 9, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 10, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs9_seed10/episode_metrics.jsonl" + }, + "git_hash": "eb3a2e1efbd2aa03a60d7f44f5e18d8fdd0f5a2d", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/context_window/train/frame_stack/uniform_u2_6/fs9/seed_10", + "wandb_unique_id": "demon-attack-fs9-uniform_u2_6-s10" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs9:obs15:stride4:seed11/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs9:obs15:stride4:seed11/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..94f92d4d05db93115920c9f657e0ae2e6a82d27d --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs9:obs15:stride4:seed11/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3e1f8ce4bd2ba402da40b5673aa85b816940f769373537e8d00f7a62ca981b16 +size 7816253 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs9:obs15:stride4:seed11/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs9:obs15:stride4:seed11/config.json new file mode 100644 index 0000000000000000000000000000000000000000..c8a31fea97220284540cc046f7173de7cc4b701a --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs9:obs15:stride4:seed11/config.json @@ -0,0 +1,255 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_frame_stack_uniform_u2_6_fs9_seed11", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 11, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-fs9-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "frame_stack", + "uniform", + "uniform_u2_6", + "fs9", + "seed11" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 15.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 9, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 11, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs9_seed11/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_frame_stack_uniform_u2_6_fs9_seed11 --train_dir results/checkpoints_factor_sweeps/demon_attack/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 11 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs9_seed11/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 11 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 36000 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-fs9-uniform_u2_6 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack frame_stack uniform uniform_u2_6 fs9 seed11 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 15.0 --screen_size 84 --noop_max 30 --frame_stack 9 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_frame_stack_uniform_u2_6_fs9_seed11", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 11, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-fs9-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "frame_stack", + "uniform", + "uniform_u2_6", + "fs9", + "seed11" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 15.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 9, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 11, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs9_seed11/episode_metrics.jsonl" + }, + "git_hash": "eb3a2e1efbd2aa03a60d7f44f5e18d8fdd0f5a2d", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/context_window/train/frame_stack/uniform_u2_6/fs9/seed_11", + "wandb_unique_id": "demon-attack-fs9-uniform_u2_6-s11" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs9:obs15:stride4:seed12/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs9:obs15:stride4:seed12/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..9885c7fb00563d770f5f827b6733556c9a9dad04 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs9:obs15:stride4:seed12/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2c1fd216d7dcc14eb3a309b87fc1afd798412db482562a85b253f4ecdc848f2b +size 7816253 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs9:obs15:stride4:seed12/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs9:obs15:stride4:seed12/config.json new file mode 100644 index 0000000000000000000000000000000000000000..667d52cbfc731c031ffeebd6eb4057de8459edec --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs9:obs15:stride4:seed12/config.json @@ -0,0 +1,255 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_frame_stack_uniform_u2_6_fs9_seed12", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 12, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-fs9-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "frame_stack", + "uniform", + "uniform_u2_6", + "fs9", + "seed12" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 15.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 9, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 12, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs9_seed12/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_frame_stack_uniform_u2_6_fs9_seed12 --train_dir results/checkpoints_factor_sweeps/demon_attack/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 12 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs9_seed12/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 12 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 36000 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-fs9-uniform_u2_6 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack frame_stack uniform uniform_u2_6 fs9 seed12 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 15.0 --screen_size 84 --noop_max 30 --frame_stack 9 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_frame_stack_uniform_u2_6_fs9_seed12", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 12, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-fs9-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "frame_stack", + "uniform", + "uniform_u2_6", + "fs9", + "seed12" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 15.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 9, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 12, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs9_seed12/episode_metrics.jsonl" + }, + "git_hash": "eb3a2e1efbd2aa03a60d7f44f5e18d8fdd0f5a2d", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/context_window/train/frame_stack/uniform_u2_6/fs9/seed_12", + "wandb_unique_id": "demon-attack-fs9-uniform_u2_6-s12" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs9:obs15:stride4:seed13/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs9:obs15:stride4:seed13/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..689b2e3abf24e112786200737e89d6bd452a2ac0 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs9:obs15:stride4:seed13/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0e09b6304fb249fd76196e9bf1a7a4c094059061c6e66f42a5ad97d72b8b32d3 +size 7816253 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs9:obs15:stride4:seed13/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs9:obs15:stride4:seed13/config.json new file mode 100644 index 0000000000000000000000000000000000000000..15b2669073d57d022c8f9c3aa7f63b6d85984956 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs9:obs15:stride4:seed13/config.json @@ -0,0 +1,255 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_frame_stack_uniform_u2_6_fs9_seed13", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 13, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-fs9-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "frame_stack", + "uniform", + "uniform_u2_6", + "fs9", + "seed13" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 15.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 9, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 13, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs9_seed13/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_frame_stack_uniform_u2_6_fs9_seed13 --train_dir results/checkpoints_factor_sweeps/demon_attack/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 13 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs9_seed13/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 13 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 36000 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-fs9-uniform_u2_6 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack frame_stack uniform uniform_u2_6 fs9 seed13 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 15.0 --screen_size 84 --noop_max 30 --frame_stack 9 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_frame_stack_uniform_u2_6_fs9_seed13", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 13, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-fs9-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "frame_stack", + "uniform", + "uniform_u2_6", + "fs9", + "seed13" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 15.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 9, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 13, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs9_seed13/episode_metrics.jsonl" + }, + "git_hash": "eb3a2e1efbd2aa03a60d7f44f5e18d8fdd0f5a2d", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/context_window/train/frame_stack/uniform_u2_6/fs9/seed_13", + "wandb_unique_id": "demon-attack-fs9-uniform_u2_6-s13" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs9:obs15:stride4:seed14/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs9:obs15:stride4:seed14/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..653b53317ceefbb6f851bacd5d9895268c23dca1 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs9:obs15:stride4:seed14/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:98569b25b769975290492fc896003e0bebec6571acd4ef75c2ee3e37ad8f02ab +size 7816253 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs9:obs15:stride4:seed14/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs9:obs15:stride4:seed14/config.json new file mode 100644 index 0000000000000000000000000000000000000000..bea5046e3f1dffb2f2283d9035ad51af75649ba2 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:frame_stack:uniform_u2_6:fs9:obs15:stride4:seed14/config.json @@ -0,0 +1,255 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_frame_stack_uniform_u2_6_fs9_seed14", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 14, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-fs9-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "frame_stack", + "uniform", + "uniform_u2_6", + "fs9", + "seed14" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 15.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 9, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 14, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs9_seed14/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_frame_stack_uniform_u2_6_fs9_seed14 --train_dir results/checkpoints_factor_sweeps/demon_attack/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 14 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs9_seed14/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 14 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 36000 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-fs9-uniform_u2_6 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack frame_stack uniform uniform_u2_6 fs9 seed14 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 15.0 --screen_size 84 --noop_max 30 --frame_stack 9 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_frame_stack_uniform_u2_6_fs9_seed14", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 14, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-fs9-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "frame_stack", + "uniform", + "uniform_u2_6", + "fs9", + "seed14" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 15.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 9, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 14, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/context_window/demon_attack_frame_stack_uniform_u2_6_fs9_seed14/episode_metrics.jsonl" + }, + "git_hash": "eb3a2e1efbd2aa03a60d7f44f5e18d8fdd0f5a2d", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/context_window/train/frame_stack/uniform_u2_6/fs9/seed_14", + "wandb_unique_id": "demon-attack-fs9-uniform_u2_6-s14" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs10:stride6:seed10/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs10:stride6:seed10/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..45f10135bf86fa3aed7b1e909330acc0f89a646f --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs10:stride6:seed10/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a37790d86a0f91f2d04b1f3c66f6c18542a2d43ae5b84ff6c51c6d72cf5ca808 +size 7210813 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs10:stride6:seed10/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs10:stride6:seed10/config.json new file mode 100644 index 0000000000000000000000000000000000000000..26f96c5b06650ef3ff644117556feab2ae51ff83 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs10:stride6:seed10/config.json @@ -0,0 +1,253 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_fixed_l4_stride6_seed10", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 10, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride6-fixed_l4", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "fixed", + "fixed_l4", + "stride6", + "seed10" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 10.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": null, + "max_latency_ms": null, + "latency_seed": null, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride6_seed10/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_observation_stride_fixed_l4_stride6_seed10 --train_dir results/checkpoints_factor_sweeps/demon_attack/observation_stride --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 10 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride6_seed10/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type fixed --fixed-latency-ms 66.66666666666667 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-stride6-fixed_l4 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack observation_stride fixed fixed_l4 stride6 seed10 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 10.0 --screen_size 84 --noop_max 30 --frame_stack 4 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_fixed_l4_stride6_seed10", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 10, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride6-fixed_l4", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "fixed", + "fixed_l4", + "stride6", + "seed10" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 10.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride6_seed10/episode_metrics.jsonl" + }, + "git_hash": "a81783762c1f965cc4a9d2eb7f028b47ab17a859", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/observation_stride/train/observation_stride/fixed_l4/stride6/seed_10", + "wandb_unique_id": "demon-attack-stride6-fixed_l4-s10" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs10:stride6:seed11/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs10:stride6:seed11/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..353110d1c7bebf3409720370f134f76e06ef0c39 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs10:stride6:seed11/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:85245b9059d6aed7753571c1cefbb3d4045e4a63d66ff1d589fa22e729e39f7d +size 7210813 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs10:stride6:seed11/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs10:stride6:seed11/config.json new file mode 100644 index 0000000000000000000000000000000000000000..3a4edf0751dd5352148cbb98a4682fdf38896d04 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs10:stride6:seed11/config.json @@ -0,0 +1,253 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_fixed_l4_stride6_seed11", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 11, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride6-fixed_l4", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "fixed", + "fixed_l4", + "stride6", + "seed11" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 10.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": null, + "max_latency_ms": null, + "latency_seed": null, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride6_seed11/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_observation_stride_fixed_l4_stride6_seed11 --train_dir results/checkpoints_factor_sweeps/demon_attack/observation_stride --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 11 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride6_seed11/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type fixed --fixed-latency-ms 66.66666666666667 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-stride6-fixed_l4 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack observation_stride fixed fixed_l4 stride6 seed11 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 10.0 --screen_size 84 --noop_max 30 --frame_stack 4 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_fixed_l4_stride6_seed11", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 11, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride6-fixed_l4", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "fixed", + "fixed_l4", + "stride6", + "seed11" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 10.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride6_seed11/episode_metrics.jsonl" + }, + "git_hash": "a81783762c1f965cc4a9d2eb7f028b47ab17a859", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/observation_stride/train/observation_stride/fixed_l4/stride6/seed_11", + "wandb_unique_id": "demon-attack-stride6-fixed_l4-s11" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs10:stride6:seed12/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs10:stride6:seed12/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..6ad701bc93d2c3c328629dbb7bb47fe3fa584a99 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs10:stride6:seed12/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61c781e90e3d419666b1b9d34025ff377a1660d9482129fb1d7eade61220660a +size 7210813 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs10:stride6:seed12/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs10:stride6:seed12/config.json new file mode 100644 index 0000000000000000000000000000000000000000..201c3e1973f13c3a4b15a9faa5bc0d53da67a8b7 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs10:stride6:seed12/config.json @@ -0,0 +1,253 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_fixed_l4_stride6_seed12", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 12, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride6-fixed_l4", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "fixed", + "fixed_l4", + "stride6", + "seed12" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 10.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": null, + "max_latency_ms": null, + "latency_seed": null, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride6_seed12/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_observation_stride_fixed_l4_stride6_seed12 --train_dir results/checkpoints_factor_sweeps/demon_attack/observation_stride --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 12 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride6_seed12/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type fixed --fixed-latency-ms 66.66666666666667 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-stride6-fixed_l4 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack observation_stride fixed fixed_l4 stride6 seed12 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 10.0 --screen_size 84 --noop_max 30 --frame_stack 4 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_fixed_l4_stride6_seed12", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 12, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride6-fixed_l4", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "fixed", + "fixed_l4", + "stride6", + "seed12" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 10.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride6_seed12/episode_metrics.jsonl" + }, + "git_hash": "a81783762c1f965cc4a9d2eb7f028b47ab17a859", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/observation_stride/train/observation_stride/fixed_l4/stride6/seed_12", + "wandb_unique_id": "demon-attack-stride6-fixed_l4-s12" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs10:stride6:seed13/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs10:stride6:seed13/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..ef85ef44c8f4620134a182af38d75f860255ab0b --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs10:stride6:seed13/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:13b6894fe3b1857731993bcf6d047016f9ad6ab2878fa2e8194aea79650fb1e4 +size 7210813 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs10:stride6:seed13/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs10:stride6:seed13/config.json new file mode 100644 index 0000000000000000000000000000000000000000..e419885425f43c39405327d55918c4a4c6467837 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs10:stride6:seed13/config.json @@ -0,0 +1,253 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_fixed_l4_stride6_seed13", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 13, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride6-fixed_l4", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "fixed", + "fixed_l4", + "stride6", + "seed13" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 10.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": null, + "max_latency_ms": null, + "latency_seed": null, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride6_seed13/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_observation_stride_fixed_l4_stride6_seed13 --train_dir results/checkpoints_factor_sweeps/demon_attack/observation_stride --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 13 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride6_seed13/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type fixed --fixed-latency-ms 66.66666666666667 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-stride6-fixed_l4 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack observation_stride fixed fixed_l4 stride6 seed13 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 10.0 --screen_size 84 --noop_max 30 --frame_stack 4 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_fixed_l4_stride6_seed13", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 13, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride6-fixed_l4", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "fixed", + "fixed_l4", + "stride6", + "seed13" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 10.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride6_seed13/episode_metrics.jsonl" + }, + "git_hash": "a81783762c1f965cc4a9d2eb7f028b47ab17a859", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/observation_stride/train/observation_stride/fixed_l4/stride6/seed_13", + "wandb_unique_id": "demon-attack-stride6-fixed_l4-s13" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs10:stride6:seed14/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs10:stride6:seed14/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..923ef24e3df8310abbc23355f7f8ebf8142082b6 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs10:stride6:seed14/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:578cbde80a7c82b44f38e8bea2ba8082ed84f5e36a8b2c770596c922595224b8 +size 7210813 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs10:stride6:seed14/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs10:stride6:seed14/config.json new file mode 100644 index 0000000000000000000000000000000000000000..0ed048969b79ad6db460522d4d917d66ad839bb8 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs10:stride6:seed14/config.json @@ -0,0 +1,253 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_fixed_l4_stride6_seed14", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 14, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride6-fixed_l4", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "fixed", + "fixed_l4", + "stride6", + "seed14" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 10.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": null, + "max_latency_ms": null, + "latency_seed": null, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride6_seed14/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_observation_stride_fixed_l4_stride6_seed14 --train_dir results/checkpoints_factor_sweeps/demon_attack/observation_stride --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 14 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride6_seed14/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type fixed --fixed-latency-ms 66.66666666666667 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-stride6-fixed_l4 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack observation_stride fixed fixed_l4 stride6 seed14 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 10.0 --screen_size 84 --noop_max 30 --frame_stack 4 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_fixed_l4_stride6_seed14", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 14, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride6-fixed_l4", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "fixed", + "fixed_l4", + "stride6", + "seed14" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 10.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride6_seed14/episode_metrics.jsonl" + }, + "git_hash": "a81783762c1f965cc4a9d2eb7f028b47ab17a859", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/observation_stride/train/observation_stride/fixed_l4/stride6/seed_14", + "wandb_unique_id": "demon-attack-stride6-fixed_l4-s14" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs12:stride5:seed10/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs12:stride5:seed10/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..44553b8bf6ea4886f376c1bd4bbbcf64f764bc5b --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs12:stride5:seed10/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:17025380620602ead9d0b9838f22d62051274fc298eff0848ba58ed242f01658 +size 7210813 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs12:stride5:seed10/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs12:stride5:seed10/config.json new file mode 100644 index 0000000000000000000000000000000000000000..dd456df0c3983b5cf04809248e5ff3f7cc7dd469 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs12:stride5:seed10/config.json @@ -0,0 +1,253 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_fixed_l4_stride5_seed10", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 10, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride5-fixed_l4", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "fixed", + "fixed_l4", + "stride5", + "seed10" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 12.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": null, + "max_latency_ms": null, + "latency_seed": null, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride5_seed10/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_observation_stride_fixed_l4_stride5_seed10 --train_dir results/checkpoints_factor_sweeps/demon_attack/observation_stride --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 10 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride5_seed10/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type fixed --fixed-latency-ms 66.66666666666667 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-stride5-fixed_l4 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack observation_stride fixed fixed_l4 stride5 seed10 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 12.0 --screen_size 84 --noop_max 30 --frame_stack 4 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_fixed_l4_stride5_seed10", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 10, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride5-fixed_l4", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "fixed", + "fixed_l4", + "stride5", + "seed10" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 12.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride5_seed10/episode_metrics.jsonl" + }, + "git_hash": "a81783762c1f965cc4a9d2eb7f028b47ab17a859", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/observation_stride/train/observation_stride/fixed_l4/stride5/seed_10", + "wandb_unique_id": "demon-attack-stride5-fixed_l4-s10" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs12:stride5:seed11/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs12:stride5:seed11/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..19dd913f1e03c65cd60995e8cb95ef2b80745484 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs12:stride5:seed11/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:de00b0dc70b2795df4aec654d7cfe4fbf3eb336cb08d25de3a73938d8acb0be0 +size 7210813 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs12:stride5:seed11/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs12:stride5:seed11/config.json new file mode 100644 index 0000000000000000000000000000000000000000..17b359af44eeb4eb9f8d12a7645785f74dd495f7 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs12:stride5:seed11/config.json @@ -0,0 +1,253 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_fixed_l4_stride5_seed11", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 11, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride5-fixed_l4", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "fixed", + "fixed_l4", + "stride5", + "seed11" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 12.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": null, + "max_latency_ms": null, + "latency_seed": null, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride5_seed11/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_observation_stride_fixed_l4_stride5_seed11 --train_dir results/checkpoints_factor_sweeps/demon_attack/observation_stride --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 11 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride5_seed11/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type fixed --fixed-latency-ms 66.66666666666667 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-stride5-fixed_l4 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack observation_stride fixed fixed_l4 stride5 seed11 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 12.0 --screen_size 84 --noop_max 30 --frame_stack 4 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_fixed_l4_stride5_seed11", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 11, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride5-fixed_l4", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "fixed", + "fixed_l4", + "stride5", + "seed11" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 12.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride5_seed11/episode_metrics.jsonl" + }, + "git_hash": "a81783762c1f965cc4a9d2eb7f028b47ab17a859", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/observation_stride/train/observation_stride/fixed_l4/stride5/seed_11", + "wandb_unique_id": "demon-attack-stride5-fixed_l4-s11" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs12:stride5:seed12/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs12:stride5:seed12/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..b10b0ec17105b0c8c9a4a62eadd4b96754de1eed --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs12:stride5:seed12/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7b1ed96c34da1f93dcfb831c0d62a1a01b930d97c64dc13cea30c665b4bb274c +size 7210813 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs12:stride5:seed12/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs12:stride5:seed12/config.json new file mode 100644 index 0000000000000000000000000000000000000000..588b860f41f3c738c2e0d7efbafeef418e5eccea --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs12:stride5:seed12/config.json @@ -0,0 +1,253 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_fixed_l4_stride5_seed12", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 12, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride5-fixed_l4", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "fixed", + "fixed_l4", + "stride5", + "seed12" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 12.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": null, + "max_latency_ms": null, + "latency_seed": null, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride5_seed12/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_observation_stride_fixed_l4_stride5_seed12 --train_dir results/checkpoints_factor_sweeps/demon_attack/observation_stride --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 12 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride5_seed12/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type fixed --fixed-latency-ms 66.66666666666667 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-stride5-fixed_l4 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack observation_stride fixed fixed_l4 stride5 seed12 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 12.0 --screen_size 84 --noop_max 30 --frame_stack 4 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_fixed_l4_stride5_seed12", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 12, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride5-fixed_l4", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "fixed", + "fixed_l4", + "stride5", + "seed12" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 12.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride5_seed12/episode_metrics.jsonl" + }, + "git_hash": "a81783762c1f965cc4a9d2eb7f028b47ab17a859", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/observation_stride/train/observation_stride/fixed_l4/stride5/seed_12", + "wandb_unique_id": "demon-attack-stride5-fixed_l4-s12" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs12:stride5:seed13/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs12:stride5:seed13/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..0087b3de211d648ee5be2abc43b4819185b88cf2 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs12:stride5:seed13/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7c4c7a23f1aaf1244733047a499f8e9f6f901bfd30b9a742fcca5a4e7279d95b +size 7210813 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs12:stride5:seed13/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs12:stride5:seed13/config.json new file mode 100644 index 0000000000000000000000000000000000000000..61cda67f6f1e9742875d657fdb83be317ac29ef3 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs12:stride5:seed13/config.json @@ -0,0 +1,253 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_fixed_l4_stride5_seed13", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 13, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride5-fixed_l4", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "fixed", + "fixed_l4", + "stride5", + "seed13" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 12.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": null, + "max_latency_ms": null, + "latency_seed": null, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride5_seed13/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_observation_stride_fixed_l4_stride5_seed13 --train_dir results/checkpoints_factor_sweeps/demon_attack/observation_stride --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 13 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride5_seed13/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type fixed --fixed-latency-ms 66.66666666666667 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-stride5-fixed_l4 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack observation_stride fixed fixed_l4 stride5 seed13 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 12.0 --screen_size 84 --noop_max 30 --frame_stack 4 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_fixed_l4_stride5_seed13", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 13, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride5-fixed_l4", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "fixed", + "fixed_l4", + "stride5", + "seed13" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 12.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride5_seed13/episode_metrics.jsonl" + }, + "git_hash": "a81783762c1f965cc4a9d2eb7f028b47ab17a859", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/observation_stride/train/observation_stride/fixed_l4/stride5/seed_13", + "wandb_unique_id": "demon-attack-stride5-fixed_l4-s13" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs12:stride5:seed14/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs12:stride5:seed14/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..126cb38ff179e84c77bc7825fae14fd75d5ace0f --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs12:stride5:seed14/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:94ab3e311408f02a1a3b788992cd3706e6e98cd92250a0ca06fbebe39f07ad1b +size 7210813 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs12:stride5:seed14/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs12:stride5:seed14/config.json new file mode 100644 index 0000000000000000000000000000000000000000..aec0ec33c9a06b102df5c077a5878c6a45852ad7 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs12:stride5:seed14/config.json @@ -0,0 +1,253 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_fixed_l4_stride5_seed14", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 14, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride5-fixed_l4", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "fixed", + "fixed_l4", + "stride5", + "seed14" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 12.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": null, + "max_latency_ms": null, + "latency_seed": null, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride5_seed14/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_observation_stride_fixed_l4_stride5_seed14 --train_dir results/checkpoints_factor_sweeps/demon_attack/observation_stride --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 14 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride5_seed14/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type fixed --fixed-latency-ms 66.66666666666667 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-stride5-fixed_l4 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack observation_stride fixed fixed_l4 stride5 seed14 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 12.0 --screen_size 84 --noop_max 30 --frame_stack 4 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_fixed_l4_stride5_seed14", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 14, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride5-fixed_l4", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "fixed", + "fixed_l4", + "stride5", + "seed14" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 12.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride5_seed14/episode_metrics.jsonl" + }, + "git_hash": "a81783762c1f965cc4a9d2eb7f028b47ab17a859", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/observation_stride/train/observation_stride/fixed_l4/stride5/seed_14", + "wandb_unique_id": "demon-attack-stride5-fixed_l4-s14" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs15:stride4:seed10/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs15:stride4:seed10/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..88ba79bd28f407c6b2d2d2ec01a009d64214eefa --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs15:stride4:seed10/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29dd9cc52b4acd3ec07a07d91839ab5cfa04178e170ac8c6d129db90a29b71f0 +size 7210813 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs15:stride4:seed10/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs15:stride4:seed10/config.json new file mode 100644 index 0000000000000000000000000000000000000000..9d3ddc0401dc0e3be1b0edaeded35b0c418d8d7a --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs15:stride4:seed10/config.json @@ -0,0 +1,253 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_fixed_l4_stride4_seed10", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 10, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride4-fixed_l4", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "fixed", + "fixed_l4", + "stride4", + "seed10" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 15.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": null, + "max_latency_ms": null, + "latency_seed": null, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride4_seed10/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_observation_stride_fixed_l4_stride4_seed10 --train_dir results/checkpoints_factor_sweeps/demon_attack/observation_stride --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 10 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride4_seed10/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type fixed --fixed-latency-ms 66.66666666666667 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-stride4-fixed_l4 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack observation_stride fixed fixed_l4 stride4 seed10 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 15.0 --screen_size 84 --noop_max 30 --frame_stack 4 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_fixed_l4_stride4_seed10", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 10, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride4-fixed_l4", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "fixed", + "fixed_l4", + "stride4", + "seed10" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 15.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride4_seed10/episode_metrics.jsonl" + }, + "git_hash": "a81783762c1f965cc4a9d2eb7f028b47ab17a859", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/observation_stride/train/observation_stride/fixed_l4/stride4/seed_10", + "wandb_unique_id": "demon-attack-stride4-fixed_l4-s10" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs15:stride4:seed11/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs15:stride4:seed11/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..dd8ee57769ec3f31bdba209bc350dcc331f7ebf8 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs15:stride4:seed11/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3ffca4a031fe9143418e06cc4c66c53a3592c866462155ff7abacc3bb357e125 +size 7210813 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs15:stride4:seed11/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs15:stride4:seed11/config.json new file mode 100644 index 0000000000000000000000000000000000000000..2d9f8bcd4f0cacdb45758fa38001243a70d7c8df --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs15:stride4:seed11/config.json @@ -0,0 +1,253 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_fixed_l4_stride4_seed11", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 11, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride4-fixed_l4", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "fixed", + "fixed_l4", + "stride4", + "seed11" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 15.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": null, + "max_latency_ms": null, + "latency_seed": null, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride4_seed11/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_observation_stride_fixed_l4_stride4_seed11 --train_dir results/checkpoints_factor_sweeps/demon_attack/observation_stride --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 11 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride4_seed11/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type fixed --fixed-latency-ms 66.66666666666667 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-stride4-fixed_l4 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack observation_stride fixed fixed_l4 stride4 seed11 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 15.0 --screen_size 84 --noop_max 30 --frame_stack 4 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_fixed_l4_stride4_seed11", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 11, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride4-fixed_l4", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "fixed", + "fixed_l4", + "stride4", + "seed11" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 15.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride4_seed11/episode_metrics.jsonl" + }, + "git_hash": "a81783762c1f965cc4a9d2eb7f028b47ab17a859", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/observation_stride/train/observation_stride/fixed_l4/stride4/seed_11", + "wandb_unique_id": "demon-attack-stride4-fixed_l4-s11" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs15:stride4:seed12/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs15:stride4:seed12/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..64b85afad84ebc101202c883b53d60862339d49c --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs15:stride4:seed12/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:233728d4b2eb7006c46ddffd56ab428aa906797a45d547bb78ac8b34fdce61fe +size 7210813 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs15:stride4:seed12/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs15:stride4:seed12/config.json new file mode 100644 index 0000000000000000000000000000000000000000..a03f81b60ff500cc467e9d654808694fb02ba13e --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs15:stride4:seed12/config.json @@ -0,0 +1,253 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_fixed_l4_stride4_seed12", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 12, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride4-fixed_l4", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "fixed", + "fixed_l4", + "stride4", + "seed12" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 15.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": null, + "max_latency_ms": null, + "latency_seed": null, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride4_seed12/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_observation_stride_fixed_l4_stride4_seed12 --train_dir results/checkpoints_factor_sweeps/demon_attack/observation_stride --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 12 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride4_seed12/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type fixed --fixed-latency-ms 66.66666666666667 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-stride4-fixed_l4 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack observation_stride fixed fixed_l4 stride4 seed12 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 15.0 --screen_size 84 --noop_max 30 --frame_stack 4 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_fixed_l4_stride4_seed12", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 12, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride4-fixed_l4", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "fixed", + "fixed_l4", + "stride4", + "seed12" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 15.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride4_seed12/episode_metrics.jsonl" + }, + "git_hash": "a81783762c1f965cc4a9d2eb7f028b47ab17a859", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/observation_stride/train/observation_stride/fixed_l4/stride4/seed_12", + "wandb_unique_id": "demon-attack-stride4-fixed_l4-s12" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs15:stride4:seed13/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs15:stride4:seed13/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..8ed1f63878acfe12c24f06d1dc0814a206bfa65d --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs15:stride4:seed13/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c1b3749dda295a322dfa86f43f9844233dbb05241fdf7689c4b37b16ccb003d7 +size 7210813 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs15:stride4:seed13/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs15:stride4:seed13/config.json new file mode 100644 index 0000000000000000000000000000000000000000..d23328131a1bb2931adac418f0da2fa0baa33174 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs15:stride4:seed13/config.json @@ -0,0 +1,253 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_fixed_l4_stride4_seed13", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 13, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride4-fixed_l4", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "fixed", + "fixed_l4", + "stride4", + "seed13" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 15.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": null, + "max_latency_ms": null, + "latency_seed": null, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride4_seed13/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_observation_stride_fixed_l4_stride4_seed13 --train_dir results/checkpoints_factor_sweeps/demon_attack/observation_stride --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 13 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride4_seed13/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type fixed --fixed-latency-ms 66.66666666666667 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-stride4-fixed_l4 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack observation_stride fixed fixed_l4 stride4 seed13 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 15.0 --screen_size 84 --noop_max 30 --frame_stack 4 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_fixed_l4_stride4_seed13", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 13, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride4-fixed_l4", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "fixed", + "fixed_l4", + "stride4", + "seed13" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 15.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride4_seed13/episode_metrics.jsonl" + }, + "git_hash": "a81783762c1f965cc4a9d2eb7f028b47ab17a859", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/observation_stride/train/observation_stride/fixed_l4/stride4/seed_13", + "wandb_unique_id": "demon-attack-stride4-fixed_l4-s13" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs15:stride4:seed14/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs15:stride4:seed14/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..9d50568705ef65ec668a0867fe8c39a0cf9f7704 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs15:stride4:seed14/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6aa74966c5ef4fe0a0a8afbc41dab65c793bfa066ff2c68451191b75117067ba +size 7210813 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs15:stride4:seed14/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs15:stride4:seed14/config.json new file mode 100644 index 0000000000000000000000000000000000000000..6a4b8dfbe360b9a1db19bdbe1b4b765bcdbe1d64 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs15:stride4:seed14/config.json @@ -0,0 +1,253 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_fixed_l4_stride4_seed14", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 14, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride4-fixed_l4", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "fixed", + "fixed_l4", + "stride4", + "seed14" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 15.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": null, + "max_latency_ms": null, + "latency_seed": null, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride4_seed14/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_observation_stride_fixed_l4_stride4_seed14 --train_dir results/checkpoints_factor_sweeps/demon_attack/observation_stride --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 14 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride4_seed14/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type fixed --fixed-latency-ms 66.66666666666667 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-stride4-fixed_l4 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack observation_stride fixed fixed_l4 stride4 seed14 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 15.0 --screen_size 84 --noop_max 30 --frame_stack 4 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_fixed_l4_stride4_seed14", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 14, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride4-fixed_l4", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "fixed", + "fixed_l4", + "stride4", + "seed14" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 15.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride4_seed14/episode_metrics.jsonl" + }, + "git_hash": "a81783762c1f965cc4a9d2eb7f028b47ab17a859", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/observation_stride/train/observation_stride/fixed_l4/stride4/seed_14", + "wandb_unique_id": "demon-attack-stride4-fixed_l4-s14" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs20:stride3:seed10/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs20:stride3:seed10/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..d5521aceaf6810b848b19a7c7094e156a28ebc58 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs20:stride3:seed10/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:02556346c57d3f6ff2769787141774f6d05916a12b65e1f0381257c7d55d2e7b +size 7210813 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs20:stride3:seed10/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs20:stride3:seed10/config.json new file mode 100644 index 0000000000000000000000000000000000000000..3280a56c322d5977eff733fc7ad567a0a5a8aaaf --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs20:stride3:seed10/config.json @@ -0,0 +1,253 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_fixed_l4_stride3_seed10", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 10, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride3-fixed_l4", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "fixed", + "fixed_l4", + "stride3", + "seed10" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 20.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": null, + "max_latency_ms": null, + "latency_seed": null, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride3_seed10/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_observation_stride_fixed_l4_stride3_seed10 --train_dir results/checkpoints_factor_sweeps/demon_attack/observation_stride --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 10 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride3_seed10/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type fixed --fixed-latency-ms 66.66666666666667 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-stride3-fixed_l4 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack observation_stride fixed fixed_l4 stride3 seed10 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 20.0 --screen_size 84 --noop_max 30 --frame_stack 4 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_fixed_l4_stride3_seed10", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 10, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride3-fixed_l4", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "fixed", + "fixed_l4", + "stride3", + "seed10" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 20.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride3_seed10/episode_metrics.jsonl" + }, + "git_hash": "a81783762c1f965cc4a9d2eb7f028b47ab17a859", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/observation_stride/train/observation_stride/fixed_l4/stride3/seed_10", + "wandb_unique_id": "demon-attack-stride3-fixed_l4-s10" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs20:stride3:seed11/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs20:stride3:seed11/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..c1c0d6010a70afabef8ed700e769d792f82d5818 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs20:stride3:seed11/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:638e84b4d93348c1aac92320f5112d8c4edf2eeb71f3b944c2d338a9b757b58f +size 7210813 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs20:stride3:seed11/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs20:stride3:seed11/config.json new file mode 100644 index 0000000000000000000000000000000000000000..8dc364cb6f2a1378e8d3e58fe812879ae7faa6a9 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs20:stride3:seed11/config.json @@ -0,0 +1,253 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_fixed_l4_stride3_seed11", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 11, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride3-fixed_l4", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "fixed", + "fixed_l4", + "stride3", + "seed11" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 20.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": null, + "max_latency_ms": null, + "latency_seed": null, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride3_seed11/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_observation_stride_fixed_l4_stride3_seed11 --train_dir results/checkpoints_factor_sweeps/demon_attack/observation_stride --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 11 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride3_seed11/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type fixed --fixed-latency-ms 66.66666666666667 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-stride3-fixed_l4 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack observation_stride fixed fixed_l4 stride3 seed11 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 20.0 --screen_size 84 --noop_max 30 --frame_stack 4 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_fixed_l4_stride3_seed11", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 11, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride3-fixed_l4", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "fixed", + "fixed_l4", + "stride3", + "seed11" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 20.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride3_seed11/episode_metrics.jsonl" + }, + "git_hash": "a81783762c1f965cc4a9d2eb7f028b47ab17a859", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/observation_stride/train/observation_stride/fixed_l4/stride3/seed_11", + "wandb_unique_id": "demon-attack-stride3-fixed_l4-s11" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs20:stride3:seed12/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs20:stride3:seed12/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..3d6ea7fa0648008a010efe0e127638bd586c0e3e --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs20:stride3:seed12/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4c1ef0455332f8b872d21b730137ff6f8ce93a107ecc9065f68ff019b7bb61fa +size 7210813 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs20:stride3:seed12/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs20:stride3:seed12/config.json new file mode 100644 index 0000000000000000000000000000000000000000..08216a815037a29910d9b82cf1e3f64314ad877b --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs20:stride3:seed12/config.json @@ -0,0 +1,253 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_fixed_l4_stride3_seed12", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 12, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride3-fixed_l4", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "fixed", + "fixed_l4", + "stride3", + "seed12" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 20.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": null, + "max_latency_ms": null, + "latency_seed": null, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride3_seed12/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_observation_stride_fixed_l4_stride3_seed12 --train_dir results/checkpoints_factor_sweeps/demon_attack/observation_stride --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 12 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride3_seed12/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type fixed --fixed-latency-ms 66.66666666666667 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-stride3-fixed_l4 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack observation_stride fixed fixed_l4 stride3 seed12 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 20.0 --screen_size 84 --noop_max 30 --frame_stack 4 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_fixed_l4_stride3_seed12", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 12, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride3-fixed_l4", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "fixed", + "fixed_l4", + "stride3", + "seed12" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 20.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride3_seed12/episode_metrics.jsonl" + }, + "git_hash": "a81783762c1f965cc4a9d2eb7f028b47ab17a859", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/observation_stride/train/observation_stride/fixed_l4/stride3/seed_12", + "wandb_unique_id": "demon-attack-stride3-fixed_l4-s12" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs20:stride3:seed13/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs20:stride3:seed13/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..b7f6da1154072498304aab06e3b66430d85fe451 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs20:stride3:seed13/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:649bc27f8a892c55bd60d0bef09e7a44768827a3d7e05ef74d49bac3d4660610 +size 7210813 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs20:stride3:seed13/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs20:stride3:seed13/config.json new file mode 100644 index 0000000000000000000000000000000000000000..276c7b18f7c2045127f47be1c31adac34af3319e --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs20:stride3:seed13/config.json @@ -0,0 +1,253 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_fixed_l4_stride3_seed13", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 13, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride3-fixed_l4", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "fixed", + "fixed_l4", + "stride3", + "seed13" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 20.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": null, + "max_latency_ms": null, + "latency_seed": null, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride3_seed13/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_observation_stride_fixed_l4_stride3_seed13 --train_dir results/checkpoints_factor_sweeps/demon_attack/observation_stride --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 13 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride3_seed13/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type fixed --fixed-latency-ms 66.66666666666667 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-stride3-fixed_l4 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack observation_stride fixed fixed_l4 stride3 seed13 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 20.0 --screen_size 84 --noop_max 30 --frame_stack 4 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_fixed_l4_stride3_seed13", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 13, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride3-fixed_l4", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "fixed", + "fixed_l4", + "stride3", + "seed13" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 20.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride3_seed13/episode_metrics.jsonl" + }, + "git_hash": "a81783762c1f965cc4a9d2eb7f028b47ab17a859", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/observation_stride/train/observation_stride/fixed_l4/stride3/seed_13", + "wandb_unique_id": "demon-attack-stride3-fixed_l4-s13" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs20:stride3:seed14/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs20:stride3:seed14/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..2e4256831fca367fff8ba27a956b69db5d363804 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs20:stride3:seed14/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d1d9f691f0661d3e1a931d892802fe59cc00cc97961da83ce9a30edaacc34ce7 +size 7210813 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs20:stride3:seed14/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs20:stride3:seed14/config.json new file mode 100644 index 0000000000000000000000000000000000000000..f62024f6c7953db7d23210ac6c4c18278d97c261 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs20:stride3:seed14/config.json @@ -0,0 +1,253 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_fixed_l4_stride3_seed14", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 14, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride3-fixed_l4", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "fixed", + "fixed_l4", + "stride3", + "seed14" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 20.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": null, + "max_latency_ms": null, + "latency_seed": null, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride3_seed14/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_observation_stride_fixed_l4_stride3_seed14 --train_dir results/checkpoints_factor_sweeps/demon_attack/observation_stride --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 14 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride3_seed14/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type fixed --fixed-latency-ms 66.66666666666667 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-stride3-fixed_l4 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack observation_stride fixed fixed_l4 stride3 seed14 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 20.0 --screen_size 84 --noop_max 30 --frame_stack 4 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_fixed_l4_stride3_seed14", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 14, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride3-fixed_l4", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "fixed", + "fixed_l4", + "stride3", + "seed14" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 20.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride3_seed14/episode_metrics.jsonl" + }, + "git_hash": "a81783762c1f965cc4a9d2eb7f028b47ab17a859", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/observation_stride/train/observation_stride/fixed_l4/stride3/seed_14", + "wandb_unique_id": "demon-attack-stride3-fixed_l4-s14" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs30:stride2:seed10/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs30:stride2:seed10/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..4cad7b6bf993c87b970ee4dbbb29fa747ccfce2b --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs30:stride2:seed10/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c4ec1632ba8ff07df6af7eb78dc0cd325fc3d51443b33e9e0f5ac657b535e1f7 +size 7210813 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs30:stride2:seed10/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs30:stride2:seed10/config.json new file mode 100644 index 0000000000000000000000000000000000000000..63b91f05c83f5b8fb376f8ed4a321dfa6d285e9c --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs30:stride2:seed10/config.json @@ -0,0 +1,253 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_fixed_l4_stride2_seed10", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 10, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride2-fixed_l4", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "fixed", + "fixed_l4", + "stride2", + "seed10" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 30.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": null, + "max_latency_ms": null, + "latency_seed": null, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride2_seed10/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_observation_stride_fixed_l4_stride2_seed10 --train_dir results/checkpoints_factor_sweeps/demon_attack/observation_stride --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 10 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride2_seed10/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type fixed --fixed-latency-ms 66.66666666666667 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-stride2-fixed_l4 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack observation_stride fixed fixed_l4 stride2 seed10 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 30.0 --screen_size 84 --noop_max 30 --frame_stack 4 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_fixed_l4_stride2_seed10", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 10, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride2-fixed_l4", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "fixed", + "fixed_l4", + "stride2", + "seed10" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 30.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride2_seed10/episode_metrics.jsonl" + }, + "git_hash": "a81783762c1f965cc4a9d2eb7f028b47ab17a859", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/observation_stride/train/observation_stride/fixed_l4/stride2/seed_10", + "wandb_unique_id": "demon-attack-stride2-fixed_l4-s10" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs30:stride2:seed11/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs30:stride2:seed11/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..f89360cff1f04a04ee06012e8713cf89a05f6393 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs30:stride2:seed11/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e1692ec11a43d459f192940c4eedc42e9744f6765d68dd6bcd48f5db6a90f1e9 +size 7210813 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs30:stride2:seed11/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs30:stride2:seed11/config.json new file mode 100644 index 0000000000000000000000000000000000000000..d2361ff3de9640de6f85fdd1bed2fb161a7c61d2 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs30:stride2:seed11/config.json @@ -0,0 +1,253 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_fixed_l4_stride2_seed11", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 11, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride2-fixed_l4", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "fixed", + "fixed_l4", + "stride2", + "seed11" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 30.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": null, + "max_latency_ms": null, + "latency_seed": null, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride2_seed11/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_observation_stride_fixed_l4_stride2_seed11 --train_dir results/checkpoints_factor_sweeps/demon_attack/observation_stride --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 11 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride2_seed11/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type fixed --fixed-latency-ms 66.66666666666667 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-stride2-fixed_l4 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack observation_stride fixed fixed_l4 stride2 seed11 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 30.0 --screen_size 84 --noop_max 30 --frame_stack 4 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_fixed_l4_stride2_seed11", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 11, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride2-fixed_l4", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "fixed", + "fixed_l4", + "stride2", + "seed11" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 30.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride2_seed11/episode_metrics.jsonl" + }, + "git_hash": "a81783762c1f965cc4a9d2eb7f028b47ab17a859", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/observation_stride/train/observation_stride/fixed_l4/stride2/seed_11", + "wandb_unique_id": "demon-attack-stride2-fixed_l4-s11" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs30:stride2:seed12/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs30:stride2:seed12/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..d89df7750ee425393636f2f4a2b959a9623e500f --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs30:stride2:seed12/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ac65d6ac064f6551848a278217563df45b80a104bf8f7b31426ee9ca638026dd +size 7210813 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs30:stride2:seed12/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs30:stride2:seed12/config.json new file mode 100644 index 0000000000000000000000000000000000000000..a61099e6ed6b91276fdead9a88090fbf6e2b467d --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs30:stride2:seed12/config.json @@ -0,0 +1,253 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_fixed_l4_stride2_seed12", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 12, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride2-fixed_l4", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "fixed", + "fixed_l4", + "stride2", + "seed12" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 30.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": null, + "max_latency_ms": null, + "latency_seed": null, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride2_seed12/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_observation_stride_fixed_l4_stride2_seed12 --train_dir results/checkpoints_factor_sweeps/demon_attack/observation_stride --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 12 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride2_seed12/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type fixed --fixed-latency-ms 66.66666666666667 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-stride2-fixed_l4 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack observation_stride fixed fixed_l4 stride2 seed12 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 30.0 --screen_size 84 --noop_max 30 --frame_stack 4 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_fixed_l4_stride2_seed12", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 12, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride2-fixed_l4", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "fixed", + "fixed_l4", + "stride2", + "seed12" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 30.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride2_seed12/episode_metrics.jsonl" + }, + "git_hash": "a81783762c1f965cc4a9d2eb7f028b47ab17a859", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/observation_stride/train/observation_stride/fixed_l4/stride2/seed_12", + "wandb_unique_id": "demon-attack-stride2-fixed_l4-s12" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs30:stride2:seed13/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs30:stride2:seed13/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..eaa4920f63c7427b94761cee7ff0977fab103865 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs30:stride2:seed13/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:93ea874de952d0bdb4d746d9b01a61f311ce4f1a6306384b722bb9bc0eeabbf8 +size 7210813 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs30:stride2:seed13/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs30:stride2:seed13/config.json new file mode 100644 index 0000000000000000000000000000000000000000..12fff346892ea16210e408f4dce7340ce8edaa2b --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs30:stride2:seed13/config.json @@ -0,0 +1,253 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_fixed_l4_stride2_seed13", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 13, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride2-fixed_l4", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "fixed", + "fixed_l4", + "stride2", + "seed13" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 30.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": null, + "max_latency_ms": null, + "latency_seed": null, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride2_seed13/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_observation_stride_fixed_l4_stride2_seed13 --train_dir results/checkpoints_factor_sweeps/demon_attack/observation_stride --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 13 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride2_seed13/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type fixed --fixed-latency-ms 66.66666666666667 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-stride2-fixed_l4 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack observation_stride fixed fixed_l4 stride2 seed13 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 30.0 --screen_size 84 --noop_max 30 --frame_stack 4 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_fixed_l4_stride2_seed13", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 13, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride2-fixed_l4", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "fixed", + "fixed_l4", + "stride2", + "seed13" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 30.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride2_seed13/episode_metrics.jsonl" + }, + "git_hash": "a81783762c1f965cc4a9d2eb7f028b47ab17a859", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/observation_stride/train/observation_stride/fixed_l4/stride2/seed_13", + "wandb_unique_id": "demon-attack-stride2-fixed_l4-s13" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs30:stride2:seed14/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs30:stride2:seed14/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..bed0d54b62a7196909543d099199d9cdc4696bac --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs30:stride2:seed14/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f49c7c974c74fbeb08a382f1a7eb602d3cd974873eeecaa9edec871879b5d1ec +size 7210813 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs30:stride2:seed14/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs30:stride2:seed14/config.json new file mode 100644 index 0000000000000000000000000000000000000000..9e981d4fd6c21949e4e619da7845a8c4f56b89b1 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs30:stride2:seed14/config.json @@ -0,0 +1,253 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_fixed_l4_stride2_seed14", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 14, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride2-fixed_l4", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "fixed", + "fixed_l4", + "stride2", + "seed14" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 30.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": null, + "max_latency_ms": null, + "latency_seed": null, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride2_seed14/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_observation_stride_fixed_l4_stride2_seed14 --train_dir results/checkpoints_factor_sweeps/demon_attack/observation_stride --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 14 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride2_seed14/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type fixed --fixed-latency-ms 66.66666666666667 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-stride2-fixed_l4 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack observation_stride fixed fixed_l4 stride2 seed14 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 30.0 --screen_size 84 --noop_max 30 --frame_stack 4 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_fixed_l4_stride2_seed14", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 14, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride2-fixed_l4", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "fixed", + "fixed_l4", + "stride2", + "seed14" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 30.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride2_seed14/episode_metrics.jsonl" + }, + "git_hash": "a81783762c1f965cc4a9d2eb7f028b47ab17a859", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/observation_stride/train/observation_stride/fixed_l4/stride2/seed_14", + "wandb_unique_id": "demon-attack-stride2-fixed_l4-s14" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs6:stride10:seed10/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs6:stride10:seed10/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..d9214e75aeb0e64e2cbdf3e86c979b95c1641a48 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs6:stride10:seed10/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:063c89b2f79f780b2de10d80edd1ee710d9a649c8831f160cda61555dc0aacf1 +size 7210813 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs6:stride10:seed10/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs6:stride10:seed10/config.json new file mode 100644 index 0000000000000000000000000000000000000000..e5b9a59a4bf413c970f847fbe8f3bf9db602fae9 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs6:stride10:seed10/config.json @@ -0,0 +1,253 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_fixed_l4_stride10_seed10", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 10, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride10-fixed_l4", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "fixed", + "fixed_l4", + "stride10", + "seed10" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 6.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": null, + "max_latency_ms": null, + "latency_seed": null, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride10_seed10/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_observation_stride_fixed_l4_stride10_seed10 --train_dir results/checkpoints_factor_sweeps/demon_attack/observation_stride --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 10 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride10_seed10/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type fixed --fixed-latency-ms 66.66666666666667 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 36000 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-stride10-fixed_l4 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack observation_stride fixed fixed_l4 stride10 seed10 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 6.0 --screen_size 84 --noop_max 30 --frame_stack 4 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_fixed_l4_stride10_seed10", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 10, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride10-fixed_l4", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "fixed", + "fixed_l4", + "stride10", + "seed10" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 6.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride10_seed10/episode_metrics.jsonl" + }, + "git_hash": "eb3a2e1efbd2aa03a60d7f44f5e18d8fdd0f5a2d", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/observation_stride/train/observation_stride/fixed_l4/stride10/seed_10", + "wandb_unique_id": "demon-attack-stride10-fixed_l4-s10" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs6:stride10:seed11/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs6:stride10:seed11/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..f90c27e523596c431c2b9c5b8c0bd55e6f57e0d4 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs6:stride10:seed11/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:adfb95093f86fda6a9e3448cd7486bbec2149b6a600544933c0c15e95a9c5e04 +size 7210813 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs6:stride10:seed11/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs6:stride10:seed11/config.json new file mode 100644 index 0000000000000000000000000000000000000000..35899301d950de15d87c9c769728ac55fde202fd --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs6:stride10:seed11/config.json @@ -0,0 +1,253 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_fixed_l4_stride10_seed11", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 11, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride10-fixed_l4", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "fixed", + "fixed_l4", + "stride10", + "seed11" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 6.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": null, + "max_latency_ms": null, + "latency_seed": null, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride10_seed11/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_observation_stride_fixed_l4_stride10_seed11 --train_dir results/checkpoints_factor_sweeps/demon_attack/observation_stride --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 11 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride10_seed11/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type fixed --fixed-latency-ms 66.66666666666667 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 36000 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-stride10-fixed_l4 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack observation_stride fixed fixed_l4 stride10 seed11 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 6.0 --screen_size 84 --noop_max 30 --frame_stack 4 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_fixed_l4_stride10_seed11", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 11, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride10-fixed_l4", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "fixed", + "fixed_l4", + "stride10", + "seed11" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 6.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride10_seed11/episode_metrics.jsonl" + }, + "git_hash": "eb3a2e1efbd2aa03a60d7f44f5e18d8fdd0f5a2d", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/observation_stride/train/observation_stride/fixed_l4/stride10/seed_11", + "wandb_unique_id": "demon-attack-stride10-fixed_l4-s11" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs6:stride10:seed12/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs6:stride10:seed12/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..e058a346387f82d9532f40211a2e8811e5148fab --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs6:stride10:seed12/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e7f8ac353abdd258adeef8fa42df1d14497b49da8303595888773ec1c64cfaf0 +size 7210813 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs6:stride10:seed12/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs6:stride10:seed12/config.json new file mode 100644 index 0000000000000000000000000000000000000000..b6c88c981495b075d1ad7077fcba28a81a4c1ef6 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs6:stride10:seed12/config.json @@ -0,0 +1,253 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_fixed_l4_stride10_seed12", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 12, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride10-fixed_l4", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "fixed", + "fixed_l4", + "stride10", + "seed12" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 6.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": null, + "max_latency_ms": null, + "latency_seed": null, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride10_seed12/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_observation_stride_fixed_l4_stride10_seed12 --train_dir results/checkpoints_factor_sweeps/demon_attack/observation_stride --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 12 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride10_seed12/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type fixed --fixed-latency-ms 66.66666666666667 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 36000 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-stride10-fixed_l4 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack observation_stride fixed fixed_l4 stride10 seed12 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 6.0 --screen_size 84 --noop_max 30 --frame_stack 4 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_fixed_l4_stride10_seed12", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 12, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride10-fixed_l4", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "fixed", + "fixed_l4", + "stride10", + "seed12" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 6.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride10_seed12/episode_metrics.jsonl" + }, + "git_hash": "eb3a2e1efbd2aa03a60d7f44f5e18d8fdd0f5a2d", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/observation_stride/train/observation_stride/fixed_l4/stride10/seed_12", + "wandb_unique_id": "demon-attack-stride10-fixed_l4-s12" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs6:stride10:seed13/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs6:stride10:seed13/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..322c07b79ab435c74bfb0a12606b4275d37f10ee --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs6:stride10:seed13/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aef34b341e12b788f94eb2cec05646cc59dce7346d9e3b8ff71022c93de9c71d +size 7210813 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs6:stride10:seed13/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs6:stride10:seed13/config.json new file mode 100644 index 0000000000000000000000000000000000000000..61d3a6d5f6e051043a7a106f3b6ee29fde7f1f93 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs6:stride10:seed13/config.json @@ -0,0 +1,253 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_fixed_l4_stride10_seed13", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 13, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride10-fixed_l4", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "fixed", + "fixed_l4", + "stride10", + "seed13" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 6.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": null, + "max_latency_ms": null, + "latency_seed": null, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride10_seed13/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_observation_stride_fixed_l4_stride10_seed13 --train_dir results/checkpoints_factor_sweeps/demon_attack/observation_stride --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 13 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride10_seed13/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type fixed --fixed-latency-ms 66.66666666666667 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 36000 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-stride10-fixed_l4 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack observation_stride fixed fixed_l4 stride10 seed13 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 6.0 --screen_size 84 --noop_max 30 --frame_stack 4 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_fixed_l4_stride10_seed13", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 13, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride10-fixed_l4", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "fixed", + "fixed_l4", + "stride10", + "seed13" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 6.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride10_seed13/episode_metrics.jsonl" + }, + "git_hash": "eb3a2e1efbd2aa03a60d7f44f5e18d8fdd0f5a2d", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/observation_stride/train/observation_stride/fixed_l4/stride10/seed_13", + "wandb_unique_id": "demon-attack-stride10-fixed_l4-s13" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs6:stride10:seed14/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs6:stride10:seed14/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..15ebe000e15373e9e14d58aad38625df4dc97f3c --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs6:stride10:seed14/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:48607ae6635b791520ab5357679296abb28daa6e043ec779a406d251b8113f9e +size 7210813 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs6:stride10:seed14/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs6:stride10:seed14/config.json new file mode 100644 index 0000000000000000000000000000000000000000..b2643c01bbff7984380deb2e100569331504e161 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs6:stride10:seed14/config.json @@ -0,0 +1,253 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_fixed_l4_stride10_seed14", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 14, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride10-fixed_l4", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "fixed", + "fixed_l4", + "stride10", + "seed14" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 6.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": null, + "max_latency_ms": null, + "latency_seed": null, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride10_seed14/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_observation_stride_fixed_l4_stride10_seed14 --train_dir results/checkpoints_factor_sweeps/demon_attack/observation_stride --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 14 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride10_seed14/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type fixed --fixed-latency-ms 66.66666666666667 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 36000 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-stride10-fixed_l4 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack observation_stride fixed fixed_l4 stride10 seed14 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 6.0 --screen_size 84 --noop_max 30 --frame_stack 4 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_fixed_l4_stride10_seed14", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 14, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride10-fixed_l4", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "fixed", + "fixed_l4", + "stride10", + "seed14" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 6.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride10_seed14/episode_metrics.jsonl" + }, + "git_hash": "eb3a2e1efbd2aa03a60d7f44f5e18d8fdd0f5a2d", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/observation_stride/train/observation_stride/fixed_l4/stride10/seed_14", + "wandb_unique_id": "demon-attack-stride10-fixed_l4-s14" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs6p66667:stride9:seed10/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs6p66667:stride9:seed10/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..60f163456dd9768f00e8d38b76549c4d203b6961 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs6p66667:stride9:seed10/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8b93d11857121828f414e1e43555fa69c0a635afebe7da364382c37f99774d5c +size 7210813 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs6p66667:stride9:seed10/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs6p66667:stride9:seed10/config.json new file mode 100644 index 0000000000000000000000000000000000000000..d6a9812427d67735523478e8070583175be3ad99 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs6p66667:stride9:seed10/config.json @@ -0,0 +1,253 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_fixed_l4_stride9_seed10", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 10, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride9-fixed_l4", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "fixed", + "fixed_l4", + "stride9", + "seed10" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 6.666666666666667, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": null, + "max_latency_ms": null, + "latency_seed": null, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride9_seed10/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_observation_stride_fixed_l4_stride9_seed10 --train_dir results/checkpoints_factor_sweeps/demon_attack/observation_stride --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 10 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride9_seed10/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type fixed --fixed-latency-ms 66.66666666666667 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 36000 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-stride9-fixed_l4 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack observation_stride fixed fixed_l4 stride9 seed10 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 6.666666666666667 --screen_size 84 --noop_max 30 --frame_stack 4 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_fixed_l4_stride9_seed10", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 10, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride9-fixed_l4", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "fixed", + "fixed_l4", + "stride9", + "seed10" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 6.666666666666667, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride9_seed10/episode_metrics.jsonl" + }, + "git_hash": "eb3a2e1efbd2aa03a60d7f44f5e18d8fdd0f5a2d", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/observation_stride/train/observation_stride/fixed_l4/stride9/seed_10", + "wandb_unique_id": "demon-attack-stride9-fixed_l4-s10" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs6p66667:stride9:seed11/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs6p66667:stride9:seed11/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..473d5cf27a73b2b3049491000f90ab0be14a54c2 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs6p66667:stride9:seed11/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b04e85bf231b785913bba067bcc5e36dedcf944838522b9885386ed75352097a +size 7210813 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs6p66667:stride9:seed11/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs6p66667:stride9:seed11/config.json new file mode 100644 index 0000000000000000000000000000000000000000..2942b50445584378b6b3aef5a18e9483516e26bc --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs6p66667:stride9:seed11/config.json @@ -0,0 +1,253 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_fixed_l4_stride9_seed11", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 11, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride9-fixed_l4", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "fixed", + "fixed_l4", + "stride9", + "seed11" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 6.666666666666667, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": null, + "max_latency_ms": null, + "latency_seed": null, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride9_seed11/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_observation_stride_fixed_l4_stride9_seed11 --train_dir results/checkpoints_factor_sweeps/demon_attack/observation_stride --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 11 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride9_seed11/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type fixed --fixed-latency-ms 66.66666666666667 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 36000 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-stride9-fixed_l4 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack observation_stride fixed fixed_l4 stride9 seed11 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 6.666666666666667 --screen_size 84 --noop_max 30 --frame_stack 4 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_fixed_l4_stride9_seed11", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 11, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride9-fixed_l4", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "fixed", + "fixed_l4", + "stride9", + "seed11" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 6.666666666666667, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride9_seed11/episode_metrics.jsonl" + }, + "git_hash": "eb3a2e1efbd2aa03a60d7f44f5e18d8fdd0f5a2d", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/observation_stride/train/observation_stride/fixed_l4/stride9/seed_11", + "wandb_unique_id": "demon-attack-stride9-fixed_l4-s11" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs6p66667:stride9:seed12/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs6p66667:stride9:seed12/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..edfb06c69381746f9e00ed36826eafc8a9af8411 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs6p66667:stride9:seed12/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f0467d5c0c51ec4b2083fd485ffbc0a9e30654577cf3f70da4a34d5bf98aeba1 +size 7210813 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs6p66667:stride9:seed12/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs6p66667:stride9:seed12/config.json new file mode 100644 index 0000000000000000000000000000000000000000..dbbdfc222dad6bbb6d6b35b9e136af900308ab67 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs6p66667:stride9:seed12/config.json @@ -0,0 +1,253 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_fixed_l4_stride9_seed12", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 12, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride9-fixed_l4", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "fixed", + "fixed_l4", + "stride9", + "seed12" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 6.666666666666667, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": null, + "max_latency_ms": null, + "latency_seed": null, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride9_seed12/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_observation_stride_fixed_l4_stride9_seed12 --train_dir results/checkpoints_factor_sweeps/demon_attack/observation_stride --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 12 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride9_seed12/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type fixed --fixed-latency-ms 66.66666666666667 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 36000 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-stride9-fixed_l4 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack observation_stride fixed fixed_l4 stride9 seed12 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 6.666666666666667 --screen_size 84 --noop_max 30 --frame_stack 4 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_fixed_l4_stride9_seed12", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 12, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride9-fixed_l4", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "fixed", + "fixed_l4", + "stride9", + "seed12" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 6.666666666666667, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride9_seed12/episode_metrics.jsonl" + }, + "git_hash": "eb3a2e1efbd2aa03a60d7f44f5e18d8fdd0f5a2d", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/observation_stride/train/observation_stride/fixed_l4/stride9/seed_12", + "wandb_unique_id": "demon-attack-stride9-fixed_l4-s12" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs6p66667:stride9:seed13/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs6p66667:stride9:seed13/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..aeb906619257679373f1557bba25c710465e9b62 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs6p66667:stride9:seed13/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9862d31575867d3c73aa1f613e89d2e547746c3a98940883edcd4aeeb0da918c +size 7210813 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs6p66667:stride9:seed13/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs6p66667:stride9:seed13/config.json new file mode 100644 index 0000000000000000000000000000000000000000..30f0339fc68cb12bf2c29aa347536a83dc01dde2 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs6p66667:stride9:seed13/config.json @@ -0,0 +1,253 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_fixed_l4_stride9_seed13", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 13, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride9-fixed_l4", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "fixed", + "fixed_l4", + "stride9", + "seed13" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 6.666666666666667, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": null, + "max_latency_ms": null, + "latency_seed": null, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride9_seed13/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_observation_stride_fixed_l4_stride9_seed13 --train_dir results/checkpoints_factor_sweeps/demon_attack/observation_stride --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 13 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride9_seed13/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type fixed --fixed-latency-ms 66.66666666666667 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 36000 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-stride9-fixed_l4 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack observation_stride fixed fixed_l4 stride9 seed13 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 6.666666666666667 --screen_size 84 --noop_max 30 --frame_stack 4 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_fixed_l4_stride9_seed13", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 13, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride9-fixed_l4", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "fixed", + "fixed_l4", + "stride9", + "seed13" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 6.666666666666667, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride9_seed13/episode_metrics.jsonl" + }, + "git_hash": "eb3a2e1efbd2aa03a60d7f44f5e18d8fdd0f5a2d", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/observation_stride/train/observation_stride/fixed_l4/stride9/seed_13", + "wandb_unique_id": "demon-attack-stride9-fixed_l4-s13" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs6p66667:stride9:seed14/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs6p66667:stride9:seed14/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..a4197ef6b9714d45a4c0461acf0c29900779ec74 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs6p66667:stride9:seed14/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7f5289317c5c528546f19f2e0b84aa39e467be5d910e3c06af9686db34f955ac +size 7210813 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs6p66667:stride9:seed14/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs6p66667:stride9:seed14/config.json new file mode 100644 index 0000000000000000000000000000000000000000..aa82cf8a5a91dabbbb78ca017206d8d09f60b666 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs6p66667:stride9:seed14/config.json @@ -0,0 +1,253 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_fixed_l4_stride9_seed14", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 14, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride9-fixed_l4", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "fixed", + "fixed_l4", + "stride9", + "seed14" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 6.666666666666667, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": null, + "max_latency_ms": null, + "latency_seed": null, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride9_seed14/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_observation_stride_fixed_l4_stride9_seed14 --train_dir results/checkpoints_factor_sweeps/demon_attack/observation_stride --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 14 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride9_seed14/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type fixed --fixed-latency-ms 66.66666666666667 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 36000 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-stride9-fixed_l4 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack observation_stride fixed fixed_l4 stride9 seed14 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 6.666666666666667 --screen_size 84 --noop_max 30 --frame_stack 4 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_fixed_l4_stride9_seed14", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 14, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride9-fixed_l4", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "fixed", + "fixed_l4", + "stride9", + "seed14" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 6.666666666666667, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride9_seed14/episode_metrics.jsonl" + }, + "git_hash": "eb3a2e1efbd2aa03a60d7f44f5e18d8fdd0f5a2d", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/observation_stride/train/observation_stride/fixed_l4/stride9/seed_14", + "wandb_unique_id": "demon-attack-stride9-fixed_l4-s14" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs7p5:stride8:seed10/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs7p5:stride8:seed10/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..fb18d5f45f05a20db373e69f3225e7e9282b6067 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs7p5:stride8:seed10/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8d6c835c77e4e22f22132935ebd1188b5a8479cd6826a0caa900503fdf551290 +size 7210813 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs7p5:stride8:seed10/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs7p5:stride8:seed10/config.json new file mode 100644 index 0000000000000000000000000000000000000000..9b1404f640361f2c925333c729e84e4e284524ac --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs7p5:stride8:seed10/config.json @@ -0,0 +1,253 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_fixed_l4_stride8_seed10", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 10, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride8-fixed_l4", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "fixed", + "fixed_l4", + "stride8", + "seed10" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 7.5, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": null, + "max_latency_ms": null, + "latency_seed": null, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride8_seed10/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_observation_stride_fixed_l4_stride8_seed10 --train_dir results/checkpoints_factor_sweeps/demon_attack/observation_stride --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 10 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride8_seed10/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type fixed --fixed-latency-ms 66.66666666666667 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 36000 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-stride8-fixed_l4 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack observation_stride fixed fixed_l4 stride8 seed10 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 7.5 --screen_size 84 --noop_max 30 --frame_stack 4 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_fixed_l4_stride8_seed10", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 10, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride8-fixed_l4", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "fixed", + "fixed_l4", + "stride8", + "seed10" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 7.5, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride8_seed10/episode_metrics.jsonl" + }, + "git_hash": "eb3a2e1efbd2aa03a60d7f44f5e18d8fdd0f5a2d", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/observation_stride/train/observation_stride/fixed_l4/stride8/seed_10", + "wandb_unique_id": "demon-attack-stride8-fixed_l4-s10" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs7p5:stride8:seed11/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs7p5:stride8:seed11/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..d5d128a05f57286277dae818fe9d03959fed5f94 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs7p5:stride8:seed11/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6283e9fef71cc60ce907039e0e3f03e748b12daac41a1613bbe0241b1017b32a +size 7210813 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs7p5:stride8:seed11/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs7p5:stride8:seed11/config.json new file mode 100644 index 0000000000000000000000000000000000000000..a59752ec8f906fb40f4a9706eb45b9297bfc6aa7 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs7p5:stride8:seed11/config.json @@ -0,0 +1,253 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_fixed_l4_stride8_seed11", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 11, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride8-fixed_l4", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "fixed", + "fixed_l4", + "stride8", + "seed11" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 7.5, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": null, + "max_latency_ms": null, + "latency_seed": null, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride8_seed11/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_observation_stride_fixed_l4_stride8_seed11 --train_dir results/checkpoints_factor_sweeps/demon_attack/observation_stride --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 11 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride8_seed11/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type fixed --fixed-latency-ms 66.66666666666667 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 36000 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-stride8-fixed_l4 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack observation_stride fixed fixed_l4 stride8 seed11 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 7.5 --screen_size 84 --noop_max 30 --frame_stack 4 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_fixed_l4_stride8_seed11", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 11, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride8-fixed_l4", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "fixed", + "fixed_l4", + "stride8", + "seed11" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 7.5, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride8_seed11/episode_metrics.jsonl" + }, + "git_hash": "eb3a2e1efbd2aa03a60d7f44f5e18d8fdd0f5a2d", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/observation_stride/train/observation_stride/fixed_l4/stride8/seed_11", + "wandb_unique_id": "demon-attack-stride8-fixed_l4-s11" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs7p5:stride8:seed12/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs7p5:stride8:seed12/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..069b1b6025107412b59c3f79d0d1ce363c805cef --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs7p5:stride8:seed12/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5e53aa730e2e77a68871fb307b0276820f56005df627b8a3ebaf09d771f0fc83 +size 7210813 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs7p5:stride8:seed12/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs7p5:stride8:seed12/config.json new file mode 100644 index 0000000000000000000000000000000000000000..fe7dc6843288d2f0545c64c8220e7d088f91b687 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs7p5:stride8:seed12/config.json @@ -0,0 +1,253 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_fixed_l4_stride8_seed12", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 12, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride8-fixed_l4", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "fixed", + "fixed_l4", + "stride8", + "seed12" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 7.5, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": null, + "max_latency_ms": null, + "latency_seed": null, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride8_seed12/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_observation_stride_fixed_l4_stride8_seed12 --train_dir results/checkpoints_factor_sweeps/demon_attack/observation_stride --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 12 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride8_seed12/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type fixed --fixed-latency-ms 66.66666666666667 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 36000 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-stride8-fixed_l4 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack observation_stride fixed fixed_l4 stride8 seed12 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 7.5 --screen_size 84 --noop_max 30 --frame_stack 4 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_fixed_l4_stride8_seed12", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 12, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride8-fixed_l4", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "fixed", + "fixed_l4", + "stride8", + "seed12" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 7.5, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride8_seed12/episode_metrics.jsonl" + }, + "git_hash": "eb3a2e1efbd2aa03a60d7f44f5e18d8fdd0f5a2d", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/observation_stride/train/observation_stride/fixed_l4/stride8/seed_12", + "wandb_unique_id": "demon-attack-stride8-fixed_l4-s12" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs7p5:stride8:seed13/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs7p5:stride8:seed13/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..b81eafef6d082bf260f34370e96860e0e9a74824 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs7p5:stride8:seed13/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1d87841909ed49d2da8c7d1be2dadf609be6580a455f38854ac771946999e70e +size 7210813 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs7p5:stride8:seed13/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs7p5:stride8:seed13/config.json new file mode 100644 index 0000000000000000000000000000000000000000..02deee26950e22d215e9ce04b529cf8004c7732f --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs7p5:stride8:seed13/config.json @@ -0,0 +1,253 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_fixed_l4_stride8_seed13", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 13, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride8-fixed_l4", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "fixed", + "fixed_l4", + "stride8", + "seed13" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 7.5, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": null, + "max_latency_ms": null, + "latency_seed": null, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride8_seed13/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_observation_stride_fixed_l4_stride8_seed13 --train_dir results/checkpoints_factor_sweeps/demon_attack/observation_stride --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 13 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride8_seed13/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type fixed --fixed-latency-ms 66.66666666666667 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 36000 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-stride8-fixed_l4 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack observation_stride fixed fixed_l4 stride8 seed13 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 7.5 --screen_size 84 --noop_max 30 --frame_stack 4 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_fixed_l4_stride8_seed13", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 13, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride8-fixed_l4", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "fixed", + "fixed_l4", + "stride8", + "seed13" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 7.5, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride8_seed13/episode_metrics.jsonl" + }, + "git_hash": "eb3a2e1efbd2aa03a60d7f44f5e18d8fdd0f5a2d", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/observation_stride/train/observation_stride/fixed_l4/stride8/seed_13", + "wandb_unique_id": "demon-attack-stride8-fixed_l4-s13" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs7p5:stride8:seed14/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs7p5:stride8:seed14/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..6f781a7b45fdc8112788cd2aaf991ddf67144b21 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs7p5:stride8:seed14/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e68ec6ca02f19c20fedde88ae15420f5677f13e435cb5bd5005b5db7dbc9bd9a +size 7210813 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs7p5:stride8:seed14/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs7p5:stride8:seed14/config.json new file mode 100644 index 0000000000000000000000000000000000000000..8626d7bd01dc9ff67d822005a2b38a95ebf397ed --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs7p5:stride8:seed14/config.json @@ -0,0 +1,253 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_fixed_l4_stride8_seed14", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 14, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride8-fixed_l4", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "fixed", + "fixed_l4", + "stride8", + "seed14" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 7.5, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": null, + "max_latency_ms": null, + "latency_seed": null, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride8_seed14/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_observation_stride_fixed_l4_stride8_seed14 --train_dir results/checkpoints_factor_sweeps/demon_attack/observation_stride --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 14 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride8_seed14/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type fixed --fixed-latency-ms 66.66666666666667 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 36000 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-stride8-fixed_l4 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack observation_stride fixed fixed_l4 stride8 seed14 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 7.5 --screen_size 84 --noop_max 30 --frame_stack 4 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_fixed_l4_stride8_seed14", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 14, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride8-fixed_l4", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "fixed", + "fixed_l4", + "stride8", + "seed14" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 7.5, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride8_seed14/episode_metrics.jsonl" + }, + "git_hash": "eb3a2e1efbd2aa03a60d7f44f5e18d8fdd0f5a2d", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/observation_stride/train/observation_stride/fixed_l4/stride8/seed_14", + "wandb_unique_id": "demon-attack-stride8-fixed_l4-s14" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs8p57143:stride7:seed10/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs8p57143:stride7:seed10/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..7a4bbf74f8ee2e09eec2bb541787909113f5fd15 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs8p57143:stride7:seed10/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:905fb9d192133923ab1d83c52429f16d0ad779ea29dd5d8fb3f687a4c9485ef8 +size 7210813 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs8p57143:stride7:seed10/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs8p57143:stride7:seed10/config.json new file mode 100644 index 0000000000000000000000000000000000000000..c50ef7fe349e6dc09d3f5c881bdc5835b8106318 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs8p57143:stride7:seed10/config.json @@ -0,0 +1,253 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_fixed_l4_stride7_seed10", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 10, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride7-fixed_l4", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "fixed", + "fixed_l4", + "stride7", + "seed10" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 8.571428571428571, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": null, + "max_latency_ms": null, + "latency_seed": null, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride7_seed10/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_observation_stride_fixed_l4_stride7_seed10 --train_dir results/checkpoints_factor_sweeps/demon_attack/observation_stride --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 10 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride7_seed10/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type fixed --fixed-latency-ms 66.66666666666667 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 36000 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-stride7-fixed_l4 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack observation_stride fixed fixed_l4 stride7 seed10 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 8.571428571428571 --screen_size 84 --noop_max 30 --frame_stack 4 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_fixed_l4_stride7_seed10", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 10, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride7-fixed_l4", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "fixed", + "fixed_l4", + "stride7", + "seed10" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 8.571428571428571, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride7_seed10/episode_metrics.jsonl" + }, + "git_hash": "eb3a2e1efbd2aa03a60d7f44f5e18d8fdd0f5a2d", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/observation_stride/train/observation_stride/fixed_l4/stride7/seed_10", + "wandb_unique_id": "demon-attack-stride7-fixed_l4-s10" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs8p57143:stride7:seed11/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs8p57143:stride7:seed11/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..5ecf2308382735bead53ed7b0b2755865c14012f --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs8p57143:stride7:seed11/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:da963df7f35fcdebe72a35bbd4e7ad4b8e132fe75e5c257f1ec0c0f58e2f8ba0 +size 7210813 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs8p57143:stride7:seed11/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs8p57143:stride7:seed11/config.json new file mode 100644 index 0000000000000000000000000000000000000000..75a45a05b13484803cbb4c3ecd48b7ae444eb42d --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs8p57143:stride7:seed11/config.json @@ -0,0 +1,253 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_fixed_l4_stride7_seed11", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 11, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride7-fixed_l4", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "fixed", + "fixed_l4", + "stride7", + "seed11" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 8.571428571428571, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": null, + "max_latency_ms": null, + "latency_seed": null, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride7_seed11/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_observation_stride_fixed_l4_stride7_seed11 --train_dir results/checkpoints_factor_sweeps/demon_attack/observation_stride --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 11 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride7_seed11/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type fixed --fixed-latency-ms 66.66666666666667 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 36000 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-stride7-fixed_l4 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack observation_stride fixed fixed_l4 stride7 seed11 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 8.571428571428571 --screen_size 84 --noop_max 30 --frame_stack 4 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_fixed_l4_stride7_seed11", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 11, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride7-fixed_l4", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "fixed", + "fixed_l4", + "stride7", + "seed11" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 8.571428571428571, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride7_seed11/episode_metrics.jsonl" + }, + "git_hash": "eb3a2e1efbd2aa03a60d7f44f5e18d8fdd0f5a2d", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/observation_stride/train/observation_stride/fixed_l4/stride7/seed_11", + "wandb_unique_id": "demon-attack-stride7-fixed_l4-s11" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs8p57143:stride7:seed12/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs8p57143:stride7:seed12/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..b66d87e6575ff241852d14c0cb2fa0e9525a88eb --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs8p57143:stride7:seed12/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:944b5d385865773c6ad24f701f58c8955a6a0f878aad088bb751326d415a8ce0 +size 7210813 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs8p57143:stride7:seed12/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs8p57143:stride7:seed12/config.json new file mode 100644 index 0000000000000000000000000000000000000000..9c02a65d119890ce9adb15800c7575200e72e58a --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs8p57143:stride7:seed12/config.json @@ -0,0 +1,253 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_fixed_l4_stride7_seed12", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 12, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride7-fixed_l4", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "fixed", + "fixed_l4", + "stride7", + "seed12" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 8.571428571428571, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": null, + "max_latency_ms": null, + "latency_seed": null, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride7_seed12/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_observation_stride_fixed_l4_stride7_seed12 --train_dir results/checkpoints_factor_sweeps/demon_attack/observation_stride --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 12 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride7_seed12/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type fixed --fixed-latency-ms 66.66666666666667 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 36000 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-stride7-fixed_l4 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack observation_stride fixed fixed_l4 stride7 seed12 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 8.571428571428571 --screen_size 84 --noop_max 30 --frame_stack 4 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_fixed_l4_stride7_seed12", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 12, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride7-fixed_l4", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "fixed", + "fixed_l4", + "stride7", + "seed12" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 8.571428571428571, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride7_seed12/episode_metrics.jsonl" + }, + "git_hash": "eb3a2e1efbd2aa03a60d7f44f5e18d8fdd0f5a2d", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/observation_stride/train/observation_stride/fixed_l4/stride7/seed_12", + "wandb_unique_id": "demon-attack-stride7-fixed_l4-s12" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs8p57143:stride7:seed13/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs8p57143:stride7:seed13/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..8afa4ec3fa07799ddd24dd037bf1d421fcc864ca --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs8p57143:stride7:seed13/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0855058f6e774c97a74f50fe2c8bdb8a0782506f379714969c87c24b8db8d2c +size 7210813 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs8p57143:stride7:seed13/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs8p57143:stride7:seed13/config.json new file mode 100644 index 0000000000000000000000000000000000000000..7316b181ace37bd279e8be2ed81a303919ede26a --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs8p57143:stride7:seed13/config.json @@ -0,0 +1,253 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_fixed_l4_stride7_seed13", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 13, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride7-fixed_l4", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "fixed", + "fixed_l4", + "stride7", + "seed13" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 8.571428571428571, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": null, + "max_latency_ms": null, + "latency_seed": null, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride7_seed13/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_observation_stride_fixed_l4_stride7_seed13 --train_dir results/checkpoints_factor_sweeps/demon_attack/observation_stride --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 13 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride7_seed13/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type fixed --fixed-latency-ms 66.66666666666667 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 36000 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-stride7-fixed_l4 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack observation_stride fixed fixed_l4 stride7 seed13 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 8.571428571428571 --screen_size 84 --noop_max 30 --frame_stack 4 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_fixed_l4_stride7_seed13", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 13, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride7-fixed_l4", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "fixed", + "fixed_l4", + "stride7", + "seed13" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 8.571428571428571, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride7_seed13/episode_metrics.jsonl" + }, + "git_hash": "eb3a2e1efbd2aa03a60d7f44f5e18d8fdd0f5a2d", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/observation_stride/train/observation_stride/fixed_l4/stride7/seed_13", + "wandb_unique_id": "demon-attack-stride7-fixed_l4-s13" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs8p57143:stride7:seed14/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs8p57143:stride7:seed14/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..377f04f977670536002f6aa52ea964cb335b6877 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs8p57143:stride7:seed14/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7dd62e00afe5f19efd27e427ffbe453da1e260a659d99c20237e5db15e9360fd +size 7210813 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs8p57143:stride7:seed14/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs8p57143:stride7:seed14/config.json new file mode 100644 index 0000000000000000000000000000000000000000..8d4ed8c471918262a67126765a325a964f117f5a --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:fixed_l4:fs4:obs8p57143:stride7:seed14/config.json @@ -0,0 +1,253 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_fixed_l4_stride7_seed14", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 14, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride7-fixed_l4", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "fixed", + "fixed_l4", + "stride7", + "seed14" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 8.571428571428571, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": null, + "max_latency_ms": null, + "latency_seed": null, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride7_seed14/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_observation_stride_fixed_l4_stride7_seed14 --train_dir results/checkpoints_factor_sweeps/demon_attack/observation_stride --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 14 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride7_seed14/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type fixed --fixed-latency-ms 66.66666666666667 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 36000 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-stride7-fixed_l4 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack observation_stride fixed fixed_l4 stride7 seed14 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 8.571428571428571 --screen_size 84 --noop_max 30 --frame_stack 4 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_fixed_l4_stride7_seed14", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 14, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride7-fixed_l4", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "fixed", + "fixed_l4", + "stride7", + "seed14" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 8.571428571428571, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_fixed_l4_stride7_seed14/episode_metrics.jsonl" + }, + "git_hash": "eb3a2e1efbd2aa03a60d7f44f5e18d8fdd0f5a2d", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/observation_stride/train/observation_stride/fixed_l4/stride7/seed_14", + "wandb_unique_id": "demon-attack-stride7-fixed_l4-s14" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs10:stride6:seed10/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs10:stride6:seed10/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..29bd16a581feb666cb9ea17411a88890fdcecf03 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs10:stride6:seed10/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e9e2bc631dfdbb46e603cf4532fe17ae1451b008c1ee640c4ad493116eb255e6 +size 7210813 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs10:stride6:seed10/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs10:stride6:seed10/config.json new file mode 100644 index 0000000000000000000000000000000000000000..9c13397c9c1e40a2df159f9f975a8792b3f39744 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs10:stride6:seed10/config.json @@ -0,0 +1,255 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_uniform_u2_6_stride6_seed10", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 10, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride6-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "uniform", + "uniform_u2_6", + "stride6", + "seed10" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 10.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 10, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride6_seed10/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_observation_stride_uniform_u2_6_stride6_seed10 --train_dir results/checkpoints_factor_sweeps/demon_attack/observation_stride --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 10 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride6_seed10/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 10 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-stride6-uniform_u2_6 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack observation_stride uniform uniform_u2_6 stride6 seed10 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 10.0 --screen_size 84 --noop_max 30 --frame_stack 4 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_uniform_u2_6_stride6_seed10", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 10, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride6-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "uniform", + "uniform_u2_6", + "stride6", + "seed10" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 10.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 10, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride6_seed10/episode_metrics.jsonl" + }, + "git_hash": "a81783762c1f965cc4a9d2eb7f028b47ab17a859", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/observation_stride/train/observation_stride/uniform_u2_6/stride6/seed_10", + "wandb_unique_id": "demon-attack-stride6-uniform_u2_6-s10" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs10:stride6:seed11/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs10:stride6:seed11/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..2b700bb22d33413f1811aa577bcae3fb67c47eff --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs10:stride6:seed11/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2ab8ec96a87647b62fad8aa9e6b04b1da90e26d645fdcb5dba432cb7200b6217 +size 7210813 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs10:stride6:seed11/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs10:stride6:seed11/config.json new file mode 100644 index 0000000000000000000000000000000000000000..0f945f3828b5a297ac3a67fd7b751b5d8cecbc5f --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs10:stride6:seed11/config.json @@ -0,0 +1,255 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_uniform_u2_6_stride6_seed11", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 11, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride6-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "uniform", + "uniform_u2_6", + "stride6", + "seed11" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 10.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 11, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride6_seed11/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_observation_stride_uniform_u2_6_stride6_seed11 --train_dir results/checkpoints_factor_sweeps/demon_attack/observation_stride --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 11 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride6_seed11/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 11 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-stride6-uniform_u2_6 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack observation_stride uniform uniform_u2_6 stride6 seed11 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 10.0 --screen_size 84 --noop_max 30 --frame_stack 4 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_uniform_u2_6_stride6_seed11", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 11, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride6-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "uniform", + "uniform_u2_6", + "stride6", + "seed11" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 10.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 11, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride6_seed11/episode_metrics.jsonl" + }, + "git_hash": "a81783762c1f965cc4a9d2eb7f028b47ab17a859", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/observation_stride/train/observation_stride/uniform_u2_6/stride6/seed_11", + "wandb_unique_id": "demon-attack-stride6-uniform_u2_6-s11" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs10:stride6:seed12/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs10:stride6:seed12/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..0f0f6f1fa21c3dddc0c896bef088088f8173ef41 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs10:stride6:seed12/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ad5afdafc699afefb06fb53db9638d36cfcf0534d2344d3e189e567812000113 +size 7210813 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs10:stride6:seed12/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs10:stride6:seed12/config.json new file mode 100644 index 0000000000000000000000000000000000000000..88e57b61bd2b2f526f09c53be3f060cc116f29c0 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs10:stride6:seed12/config.json @@ -0,0 +1,255 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_uniform_u2_6_stride6_seed12", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 12, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride6-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "uniform", + "uniform_u2_6", + "stride6", + "seed12" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 10.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 12, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride6_seed12/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_observation_stride_uniform_u2_6_stride6_seed12 --train_dir results/checkpoints_factor_sweeps/demon_attack/observation_stride --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 12 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride6_seed12/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 12 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-stride6-uniform_u2_6 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack observation_stride uniform uniform_u2_6 stride6 seed12 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 10.0 --screen_size 84 --noop_max 30 --frame_stack 4 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_uniform_u2_6_stride6_seed12", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 12, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride6-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "uniform", + "uniform_u2_6", + "stride6", + "seed12" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 10.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 12, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride6_seed12/episode_metrics.jsonl" + }, + "git_hash": "a81783762c1f965cc4a9d2eb7f028b47ab17a859", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/observation_stride/train/observation_stride/uniform_u2_6/stride6/seed_12", + "wandb_unique_id": "demon-attack-stride6-uniform_u2_6-s12" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs10:stride6:seed13/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs10:stride6:seed13/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..e22010e614ac5f523ae2d0625286439aae378d79 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs10:stride6:seed13/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29bbec55c5f133c846b143187b93da5cb5ef8e95ea567f20ea62b9bbf2f14138 +size 7210813 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs10:stride6:seed13/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs10:stride6:seed13/config.json new file mode 100644 index 0000000000000000000000000000000000000000..3c92a5a1930741965862d5f320fe418a47201a34 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs10:stride6:seed13/config.json @@ -0,0 +1,255 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_uniform_u2_6_stride6_seed13", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 13, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride6-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "uniform", + "uniform_u2_6", + "stride6", + "seed13" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 10.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 13, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride6_seed13/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_observation_stride_uniform_u2_6_stride6_seed13 --train_dir results/checkpoints_factor_sweeps/demon_attack/observation_stride --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 13 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride6_seed13/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 13 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-stride6-uniform_u2_6 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack observation_stride uniform uniform_u2_6 stride6 seed13 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 10.0 --screen_size 84 --noop_max 30 --frame_stack 4 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_uniform_u2_6_stride6_seed13", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 13, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride6-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "uniform", + "uniform_u2_6", + "stride6", + "seed13" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 10.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 13, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride6_seed13/episode_metrics.jsonl" + }, + "git_hash": "a81783762c1f965cc4a9d2eb7f028b47ab17a859", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/observation_stride/train/observation_stride/uniform_u2_6/stride6/seed_13", + "wandb_unique_id": "demon-attack-stride6-uniform_u2_6-s13" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs10:stride6:seed14/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs10:stride6:seed14/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..bae3be00da997b2220a24f33b8727e84edf314cb --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs10:stride6:seed14/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d7a858011c543905ca4daf12401614cb06f36576a7f831109bdf3b1add182c17 +size 7210813 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs10:stride6:seed14/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs10:stride6:seed14/config.json new file mode 100644 index 0000000000000000000000000000000000000000..7f0dd0a85f146f235949fc8563407472564766ce --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs10:stride6:seed14/config.json @@ -0,0 +1,255 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_uniform_u2_6_stride6_seed14", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 14, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride6-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "uniform", + "uniform_u2_6", + "stride6", + "seed14" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 10.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 14, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride6_seed14/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_observation_stride_uniform_u2_6_stride6_seed14 --train_dir results/checkpoints_factor_sweeps/demon_attack/observation_stride --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 14 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride6_seed14/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 14 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-stride6-uniform_u2_6 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack observation_stride uniform uniform_u2_6 stride6 seed14 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 10.0 --screen_size 84 --noop_max 30 --frame_stack 4 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_uniform_u2_6_stride6_seed14", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 14, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride6-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "uniform", + "uniform_u2_6", + "stride6", + "seed14" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 10.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 14, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride6_seed14/episode_metrics.jsonl" + }, + "git_hash": "a81783762c1f965cc4a9d2eb7f028b47ab17a859", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/observation_stride/train/observation_stride/uniform_u2_6/stride6/seed_14", + "wandb_unique_id": "demon-attack-stride6-uniform_u2_6-s14" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs12:stride5:seed10/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs12:stride5:seed10/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..f7d0cd1a4cf8a5e03518afc3daefd171945b862b --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs12:stride5:seed10/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0db96f5a6dd898ad3e346289f47f44d44ace8e5e3c676d0e0142c1860ff48f5d +size 7210813 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs12:stride5:seed10/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs12:stride5:seed10/config.json new file mode 100644 index 0000000000000000000000000000000000000000..85d957824c26c2d88cf3b89debab12e727f87140 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs12:stride5:seed10/config.json @@ -0,0 +1,255 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_uniform_u2_6_stride5_seed10", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 10, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride5-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "uniform", + "uniform_u2_6", + "stride5", + "seed10" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 12.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 10, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride5_seed10/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_observation_stride_uniform_u2_6_stride5_seed10 --train_dir results/checkpoints_factor_sweeps/demon_attack/observation_stride --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 10 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride5_seed10/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 10 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-stride5-uniform_u2_6 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack observation_stride uniform uniform_u2_6 stride5 seed10 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 12.0 --screen_size 84 --noop_max 30 --frame_stack 4 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_uniform_u2_6_stride5_seed10", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 10, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride5-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "uniform", + "uniform_u2_6", + "stride5", + "seed10" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 12.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 10, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride5_seed10/episode_metrics.jsonl" + }, + "git_hash": "a81783762c1f965cc4a9d2eb7f028b47ab17a859", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/observation_stride/train/observation_stride/uniform_u2_6/stride5/seed_10", + "wandb_unique_id": "demon-attack-stride5-uniform_u2_6-s10" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs12:stride5:seed11/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs12:stride5:seed11/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..a78c0556e8d84954cd156eaf093a80c4d2260acc --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs12:stride5:seed11/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c01d340fb599dab89e8a1b447044512835fa229f05e9c5f177895b3d864b3202 +size 7210813 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs12:stride5:seed11/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs12:stride5:seed11/config.json new file mode 100644 index 0000000000000000000000000000000000000000..a0578883e8282cf082ac97ce71f554671de6a8de --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs12:stride5:seed11/config.json @@ -0,0 +1,255 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_uniform_u2_6_stride5_seed11", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 11, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride5-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "uniform", + "uniform_u2_6", + "stride5", + "seed11" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 12.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 11, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride5_seed11/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_observation_stride_uniform_u2_6_stride5_seed11 --train_dir results/checkpoints_factor_sweeps/demon_attack/observation_stride --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 11 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride5_seed11/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 11 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-stride5-uniform_u2_6 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack observation_stride uniform uniform_u2_6 stride5 seed11 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 12.0 --screen_size 84 --noop_max 30 --frame_stack 4 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_uniform_u2_6_stride5_seed11", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 11, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride5-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "uniform", + "uniform_u2_6", + "stride5", + "seed11" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 12.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 11, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride5_seed11/episode_metrics.jsonl" + }, + "git_hash": "a81783762c1f965cc4a9d2eb7f028b47ab17a859", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/observation_stride/train/observation_stride/uniform_u2_6/stride5/seed_11", + "wandb_unique_id": "demon-attack-stride5-uniform_u2_6-s11" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs12:stride5:seed12/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs12:stride5:seed12/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..d46002d7ca0fc8e9dfafbe7a025f30ae33d72356 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs12:stride5:seed12/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8f62b9abf21a1cab25bb4c008bce7549e68697ad3f5838851539ebcc7e2567ad +size 7210813 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs12:stride5:seed12/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs12:stride5:seed12/config.json new file mode 100644 index 0000000000000000000000000000000000000000..05bf54bbdee8bacbef0a1e96bee18852e82f1035 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs12:stride5:seed12/config.json @@ -0,0 +1,255 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_uniform_u2_6_stride5_seed12", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 12, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride5-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "uniform", + "uniform_u2_6", + "stride5", + "seed12" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 12.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 12, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride5_seed12/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_observation_stride_uniform_u2_6_stride5_seed12 --train_dir results/checkpoints_factor_sweeps/demon_attack/observation_stride --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 12 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride5_seed12/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 12 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-stride5-uniform_u2_6 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack observation_stride uniform uniform_u2_6 stride5 seed12 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 12.0 --screen_size 84 --noop_max 30 --frame_stack 4 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_uniform_u2_6_stride5_seed12", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 12, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride5-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "uniform", + "uniform_u2_6", + "stride5", + "seed12" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 12.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 12, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride5_seed12/episode_metrics.jsonl" + }, + "git_hash": "a81783762c1f965cc4a9d2eb7f028b47ab17a859", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/observation_stride/train/observation_stride/uniform_u2_6/stride5/seed_12", + "wandb_unique_id": "demon-attack-stride5-uniform_u2_6-s12" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs12:stride5:seed13/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs12:stride5:seed13/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..2410f9eca92d28e0f2e46204c9f80074c9e3b9f9 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs12:stride5:seed13/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:78646157bd60edd119a0e3b566b3f87c25d791a91eec8a7df67b3d3dc92bba63 +size 7210813 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs12:stride5:seed13/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs12:stride5:seed13/config.json new file mode 100644 index 0000000000000000000000000000000000000000..fe4af6a051c66ddded6dde727065241e4018ec47 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs12:stride5:seed13/config.json @@ -0,0 +1,255 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_uniform_u2_6_stride5_seed13", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 13, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride5-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "uniform", + "uniform_u2_6", + "stride5", + "seed13" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 12.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 13, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride5_seed13/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_observation_stride_uniform_u2_6_stride5_seed13 --train_dir results/checkpoints_factor_sweeps/demon_attack/observation_stride --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 13 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride5_seed13/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 13 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-stride5-uniform_u2_6 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack observation_stride uniform uniform_u2_6 stride5 seed13 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 12.0 --screen_size 84 --noop_max 30 --frame_stack 4 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_uniform_u2_6_stride5_seed13", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 13, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride5-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "uniform", + "uniform_u2_6", + "stride5", + "seed13" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 12.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 13, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride5_seed13/episode_metrics.jsonl" + }, + "git_hash": "a81783762c1f965cc4a9d2eb7f028b47ab17a859", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/observation_stride/train/observation_stride/uniform_u2_6/stride5/seed_13", + "wandb_unique_id": "demon-attack-stride5-uniform_u2_6-s13" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs12:stride5:seed14/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs12:stride5:seed14/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..495a8bd9c0b9422c09af74da7819d5c6d341b8fd --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs12:stride5:seed14/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:750baf89da77bf6357fd143000eb0e90b8f6dec1eac243c0fda46d0a549f341a +size 7210813 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs12:stride5:seed14/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs12:stride5:seed14/config.json new file mode 100644 index 0000000000000000000000000000000000000000..2fc426065d9ef7376604e7cb080dc82f9892b0ec --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs12:stride5:seed14/config.json @@ -0,0 +1,255 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_uniform_u2_6_stride5_seed14", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 14, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride5-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "uniform", + "uniform_u2_6", + "stride5", + "seed14" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 12.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 14, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride5_seed14/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_observation_stride_uniform_u2_6_stride5_seed14 --train_dir results/checkpoints_factor_sweeps/demon_attack/observation_stride --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 14 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride5_seed14/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 14 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-stride5-uniform_u2_6 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack observation_stride uniform uniform_u2_6 stride5 seed14 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 12.0 --screen_size 84 --noop_max 30 --frame_stack 4 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_uniform_u2_6_stride5_seed14", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 14, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride5-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "uniform", + "uniform_u2_6", + "stride5", + "seed14" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 12.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 14, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride5_seed14/episode_metrics.jsonl" + }, + "git_hash": "a81783762c1f965cc4a9d2eb7f028b47ab17a859", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/observation_stride/train/observation_stride/uniform_u2_6/stride5/seed_14", + "wandb_unique_id": "demon-attack-stride5-uniform_u2_6-s14" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs15:stride4:seed10/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs15:stride4:seed10/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..90a9a64bbbdad48bd097c462ba3299e2cbbeba34 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs15:stride4:seed10/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2c26a20fbb496986c3229bbc5c9be8ec37e138402abdd4b040a30cf6a27f6d36 +size 7210813 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs15:stride4:seed10/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs15:stride4:seed10/config.json new file mode 100644 index 0000000000000000000000000000000000000000..3d7b019861397f2510a84203df354dfda01f06c1 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs15:stride4:seed10/config.json @@ -0,0 +1,255 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_uniform_u2_6_stride4_seed10", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 10, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride4-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "uniform", + "uniform_u2_6", + "stride4", + "seed10" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 15.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 10, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride4_seed10/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_observation_stride_uniform_u2_6_stride4_seed10 --train_dir results/checkpoints_factor_sweeps/demon_attack/observation_stride --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 10 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride4_seed10/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 10 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-stride4-uniform_u2_6 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack observation_stride uniform uniform_u2_6 stride4 seed10 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 15.0 --screen_size 84 --noop_max 30 --frame_stack 4 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_uniform_u2_6_stride4_seed10", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 10, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride4-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "uniform", + "uniform_u2_6", + "stride4", + "seed10" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 15.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 10, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride4_seed10/episode_metrics.jsonl" + }, + "git_hash": "a81783762c1f965cc4a9d2eb7f028b47ab17a859", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/observation_stride/train/observation_stride/uniform_u2_6/stride4/seed_10", + "wandb_unique_id": "demon-attack-stride4-uniform_u2_6-s10" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs15:stride4:seed11/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs15:stride4:seed11/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..1bbc64038d077366c67d583ef798f904207ae76f --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs15:stride4:seed11/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5dfcfe95096213d22562ef86b6545e3cb3affad3e8ca009fbcd0418173103897 +size 7210813 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs15:stride4:seed11/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs15:stride4:seed11/config.json new file mode 100644 index 0000000000000000000000000000000000000000..64eb693ac67ab6ada3ff1d77682596b26e24a097 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs15:stride4:seed11/config.json @@ -0,0 +1,255 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_uniform_u2_6_stride4_seed11", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 11, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride4-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "uniform", + "uniform_u2_6", + "stride4", + "seed11" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 15.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 11, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride4_seed11/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_observation_stride_uniform_u2_6_stride4_seed11 --train_dir results/checkpoints_factor_sweeps/demon_attack/observation_stride --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 11 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride4_seed11/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 11 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-stride4-uniform_u2_6 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack observation_stride uniform uniform_u2_6 stride4 seed11 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 15.0 --screen_size 84 --noop_max 30 --frame_stack 4 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_uniform_u2_6_stride4_seed11", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 11, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride4-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "uniform", + "uniform_u2_6", + "stride4", + "seed11" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 15.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 11, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride4_seed11/episode_metrics.jsonl" + }, + "git_hash": "a81783762c1f965cc4a9d2eb7f028b47ab17a859", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/observation_stride/train/observation_stride/uniform_u2_6/stride4/seed_11", + "wandb_unique_id": "demon-attack-stride4-uniform_u2_6-s11" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs15:stride4:seed12/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs15:stride4:seed12/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..b66a63e37d79dc58fe82c29a518871301fe9500f --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs15:stride4:seed12/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:141f831e6b5b92520cb513469d245f41a0304f2232d96596e514008218622cda +size 7210813 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs15:stride4:seed12/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs15:stride4:seed12/config.json new file mode 100644 index 0000000000000000000000000000000000000000..374756c5a60fce708b01391354200d1bc757bd14 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs15:stride4:seed12/config.json @@ -0,0 +1,255 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_uniform_u2_6_stride4_seed12", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 12, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride4-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "uniform", + "uniform_u2_6", + "stride4", + "seed12" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 15.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 12, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride4_seed12/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_observation_stride_uniform_u2_6_stride4_seed12 --train_dir results/checkpoints_factor_sweeps/demon_attack/observation_stride --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 12 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride4_seed12/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 12 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-stride4-uniform_u2_6 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack observation_stride uniform uniform_u2_6 stride4 seed12 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 15.0 --screen_size 84 --noop_max 30 --frame_stack 4 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_uniform_u2_6_stride4_seed12", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 12, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride4-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "uniform", + "uniform_u2_6", + "stride4", + "seed12" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 15.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 12, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride4_seed12/episode_metrics.jsonl" + }, + "git_hash": "a81783762c1f965cc4a9d2eb7f028b47ab17a859", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/observation_stride/train/observation_stride/uniform_u2_6/stride4/seed_12", + "wandb_unique_id": "demon-attack-stride4-uniform_u2_6-s12" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs15:stride4:seed13/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs15:stride4:seed13/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..11326a49d7148bb04437da5a2868883bf1ab49da --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs15:stride4:seed13/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eb793434e01f0eede5baa21cacb1020f3c349c5a857081c49e406c0c63e1e2e0 +size 7210813 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs15:stride4:seed13/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs15:stride4:seed13/config.json new file mode 100644 index 0000000000000000000000000000000000000000..448e752a8ab587ccbce889b0f44c7b2abe6bee46 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs15:stride4:seed13/config.json @@ -0,0 +1,255 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_uniform_u2_6_stride4_seed13", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 13, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride4-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "uniform", + "uniform_u2_6", + "stride4", + "seed13" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 15.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 13, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride4_seed13/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_observation_stride_uniform_u2_6_stride4_seed13 --train_dir results/checkpoints_factor_sweeps/demon_attack/observation_stride --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 13 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride4_seed13/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 13 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-stride4-uniform_u2_6 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack observation_stride uniform uniform_u2_6 stride4 seed13 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 15.0 --screen_size 84 --noop_max 30 --frame_stack 4 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_uniform_u2_6_stride4_seed13", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 13, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride4-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "uniform", + "uniform_u2_6", + "stride4", + "seed13" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 15.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 13, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride4_seed13/episode_metrics.jsonl" + }, + "git_hash": "a81783762c1f965cc4a9d2eb7f028b47ab17a859", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/observation_stride/train/observation_stride/uniform_u2_6/stride4/seed_13", + "wandb_unique_id": "demon-attack-stride4-uniform_u2_6-s13" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs15:stride4:seed14/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs15:stride4:seed14/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..28207683c72067ac5cf26704e3b78f6035c49151 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs15:stride4:seed14/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:13929dccb58f4d61ca71e60a66d85812913d302c18bcf6eb739aac894a41a580 +size 7210813 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs15:stride4:seed14/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs15:stride4:seed14/config.json new file mode 100644 index 0000000000000000000000000000000000000000..99d2149328f078fc6c1a5758354b8d1ac47c838a --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs15:stride4:seed14/config.json @@ -0,0 +1,255 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_uniform_u2_6_stride4_seed14", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 14, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride4-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "uniform", + "uniform_u2_6", + "stride4", + "seed14" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 15.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 14, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride4_seed14/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_observation_stride_uniform_u2_6_stride4_seed14 --train_dir results/checkpoints_factor_sweeps/demon_attack/observation_stride --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 14 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride4_seed14/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 14 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-stride4-uniform_u2_6 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack observation_stride uniform uniform_u2_6 stride4 seed14 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 15.0 --screen_size 84 --noop_max 30 --frame_stack 4 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_uniform_u2_6_stride4_seed14", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 14, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride4-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "uniform", + "uniform_u2_6", + "stride4", + "seed14" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 15.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 14, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride4_seed14/episode_metrics.jsonl" + }, + "git_hash": "a81783762c1f965cc4a9d2eb7f028b47ab17a859", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/observation_stride/train/observation_stride/uniform_u2_6/stride4/seed_14", + "wandb_unique_id": "demon-attack-stride4-uniform_u2_6-s14" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs20:stride3:seed10/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs20:stride3:seed10/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..fede3bf9c117626942c288153cd02e2e4ad69f79 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs20:stride3:seed10/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be3abeeb949a94806241d2dfb38ac3e2ba7e4afad13575064431dd6da3efda83 +size 7210813 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs20:stride3:seed10/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs20:stride3:seed10/config.json new file mode 100644 index 0000000000000000000000000000000000000000..67a639dec6464c838be8af374ef92ab378b7da06 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs20:stride3:seed10/config.json @@ -0,0 +1,255 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_uniform_u2_6_stride3_seed10", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 10, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride3-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "uniform", + "uniform_u2_6", + "stride3", + "seed10" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 20.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 10, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride3_seed10/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_observation_stride_uniform_u2_6_stride3_seed10 --train_dir results/checkpoints_factor_sweeps/demon_attack/observation_stride --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 10 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride3_seed10/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 10 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-stride3-uniform_u2_6 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack observation_stride uniform uniform_u2_6 stride3 seed10 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 20.0 --screen_size 84 --noop_max 30 --frame_stack 4 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_uniform_u2_6_stride3_seed10", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 10, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride3-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "uniform", + "uniform_u2_6", + "stride3", + "seed10" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 20.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 10, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride3_seed10/episode_metrics.jsonl" + }, + "git_hash": "a81783762c1f965cc4a9d2eb7f028b47ab17a859", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/observation_stride/train/observation_stride/uniform_u2_6/stride3/seed_10", + "wandb_unique_id": "demon-attack-stride3-uniform_u2_6-s10" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs20:stride3:seed11/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs20:stride3:seed11/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..c5eacf5c1a3607798d4dfd057e394989b9f5b73f --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs20:stride3:seed11/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:03642c5d38d7585184c47816190ffc0206d553acd9bc1a2308c3a31510fc1d8b +size 7210813 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs20:stride3:seed11/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs20:stride3:seed11/config.json new file mode 100644 index 0000000000000000000000000000000000000000..90f3abc5269a4afc4d79526d7d1424240dafd97b --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs20:stride3:seed11/config.json @@ -0,0 +1,255 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_uniform_u2_6_stride3_seed11", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 11, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride3-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "uniform", + "uniform_u2_6", + "stride3", + "seed11" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 20.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 11, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride3_seed11/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_observation_stride_uniform_u2_6_stride3_seed11 --train_dir results/checkpoints_factor_sweeps/demon_attack/observation_stride --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 11 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride3_seed11/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 11 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-stride3-uniform_u2_6 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack observation_stride uniform uniform_u2_6 stride3 seed11 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 20.0 --screen_size 84 --noop_max 30 --frame_stack 4 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_uniform_u2_6_stride3_seed11", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 11, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride3-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "uniform", + "uniform_u2_6", + "stride3", + "seed11" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 20.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 11, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride3_seed11/episode_metrics.jsonl" + }, + "git_hash": "a81783762c1f965cc4a9d2eb7f028b47ab17a859", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/observation_stride/train/observation_stride/uniform_u2_6/stride3/seed_11", + "wandb_unique_id": "demon-attack-stride3-uniform_u2_6-s11" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs20:stride3:seed12/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs20:stride3:seed12/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..415a6bb66e5cdbac7e197f46a8d760c8fa0f50de --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs20:stride3:seed12/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8a5e82cec38916c7323f51f711d5e7d704cef32ea18f6495493cc4817b1678d1 +size 7210813 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs20:stride3:seed12/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs20:stride3:seed12/config.json new file mode 100644 index 0000000000000000000000000000000000000000..428a4a8aa4ccb1002589ee2807648e7295aad6a0 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs20:stride3:seed12/config.json @@ -0,0 +1,255 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_uniform_u2_6_stride3_seed12", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 12, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride3-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "uniform", + "uniform_u2_6", + "stride3", + "seed12" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 20.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 12, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride3_seed12/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_observation_stride_uniform_u2_6_stride3_seed12 --train_dir results/checkpoints_factor_sweeps/demon_attack/observation_stride --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 12 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride3_seed12/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 12 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-stride3-uniform_u2_6 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack observation_stride uniform uniform_u2_6 stride3 seed12 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 20.0 --screen_size 84 --noop_max 30 --frame_stack 4 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_uniform_u2_6_stride3_seed12", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 12, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride3-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "uniform", + "uniform_u2_6", + "stride3", + "seed12" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 20.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 12, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride3_seed12/episode_metrics.jsonl" + }, + "git_hash": "a81783762c1f965cc4a9d2eb7f028b47ab17a859", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/observation_stride/train/observation_stride/uniform_u2_6/stride3/seed_12", + "wandb_unique_id": "demon-attack-stride3-uniform_u2_6-s12" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs20:stride3:seed13/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs20:stride3:seed13/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..5992d4b288c143755e70dbda5b50a03ffb161c7f --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs20:stride3:seed13/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7bf2ab78e117e80aca7a3c7c17d6d3c522f50226bc2b6d4b494d5c6d3bc45543 +size 7210813 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs20:stride3:seed13/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs20:stride3:seed13/config.json new file mode 100644 index 0000000000000000000000000000000000000000..2048ea08300c52cd864c55e8611efd83ba2fa5f8 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs20:stride3:seed13/config.json @@ -0,0 +1,255 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_uniform_u2_6_stride3_seed13", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 13, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride3-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "uniform", + "uniform_u2_6", + "stride3", + "seed13" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 20.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 13, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride3_seed13/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_observation_stride_uniform_u2_6_stride3_seed13 --train_dir results/checkpoints_factor_sweeps/demon_attack/observation_stride --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 13 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride3_seed13/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 13 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-stride3-uniform_u2_6 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack observation_stride uniform uniform_u2_6 stride3 seed13 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 20.0 --screen_size 84 --noop_max 30 --frame_stack 4 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_uniform_u2_6_stride3_seed13", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 13, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride3-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "uniform", + "uniform_u2_6", + "stride3", + "seed13" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 20.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 13, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride3_seed13/episode_metrics.jsonl" + }, + "git_hash": "a81783762c1f965cc4a9d2eb7f028b47ab17a859", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/observation_stride/train/observation_stride/uniform_u2_6/stride3/seed_13", + "wandb_unique_id": "demon-attack-stride3-uniform_u2_6-s13" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs20:stride3:seed14/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs20:stride3:seed14/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..cfcefd6d1346ded48d2ea0e6aff2dddebbee146e --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs20:stride3:seed14/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ec81132777ec173ffa51f5a71eddea14533626b1e6362824ad8cc84b72e8188e +size 7210813 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs20:stride3:seed14/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs20:stride3:seed14/config.json new file mode 100644 index 0000000000000000000000000000000000000000..278196536e80a725976339238b3aee07a7f0926e --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs20:stride3:seed14/config.json @@ -0,0 +1,255 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_uniform_u2_6_stride3_seed14", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 14, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride3-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "uniform", + "uniform_u2_6", + "stride3", + "seed14" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 20.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 14, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride3_seed14/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_observation_stride_uniform_u2_6_stride3_seed14 --train_dir results/checkpoints_factor_sweeps/demon_attack/observation_stride --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 14 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride3_seed14/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 14 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-stride3-uniform_u2_6 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack observation_stride uniform uniform_u2_6 stride3 seed14 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 20.0 --screen_size 84 --noop_max 30 --frame_stack 4 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_uniform_u2_6_stride3_seed14", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 14, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride3-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "uniform", + "uniform_u2_6", + "stride3", + "seed14" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 20.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 14, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride3_seed14/episode_metrics.jsonl" + }, + "git_hash": "a81783762c1f965cc4a9d2eb7f028b47ab17a859", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/observation_stride/train/observation_stride/uniform_u2_6/stride3/seed_14", + "wandb_unique_id": "demon-attack-stride3-uniform_u2_6-s14" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs30:stride2:seed10/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs30:stride2:seed10/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..893c142e6a3380dbbcbd672fa0d2f9160bd1d151 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs30:stride2:seed10/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d9b22e7afcd1b3bdbcf2692e6244aea42e3387266be93b82a972f0901651d57e +size 7210813 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs30:stride2:seed10/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs30:stride2:seed10/config.json new file mode 100644 index 0000000000000000000000000000000000000000..15f17d8fce950c4bc74f6446f412d7cb392caab8 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs30:stride2:seed10/config.json @@ -0,0 +1,255 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_uniform_u2_6_stride2_seed10", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 10, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride2-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "uniform", + "uniform_u2_6", + "stride2", + "seed10" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 30.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 10, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride2_seed10/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_observation_stride_uniform_u2_6_stride2_seed10 --train_dir results/checkpoints_factor_sweeps/demon_attack/observation_stride --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 10 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride2_seed10/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 10 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-stride2-uniform_u2_6 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack observation_stride uniform uniform_u2_6 stride2 seed10 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 30.0 --screen_size 84 --noop_max 30 --frame_stack 4 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_uniform_u2_6_stride2_seed10", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 10, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride2-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "uniform", + "uniform_u2_6", + "stride2", + "seed10" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 30.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 10, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride2_seed10/episode_metrics.jsonl" + }, + "git_hash": "a81783762c1f965cc4a9d2eb7f028b47ab17a859", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/observation_stride/train/observation_stride/uniform_u2_6/stride2/seed_10", + "wandb_unique_id": "demon-attack-stride2-uniform_u2_6-s10" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs30:stride2:seed11/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs30:stride2:seed11/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..9f155c3e142ebd9cc67122920ba604458503ec4f --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs30:stride2:seed11/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:774c19ba5d8014d87b1131c355f52be31180bf5e4e9fc480be08e32162bb4ac0 +size 7210813 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs30:stride2:seed11/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs30:stride2:seed11/config.json new file mode 100644 index 0000000000000000000000000000000000000000..db59ca398d687c6c7d2e35b3c9442bb8b0fba49b --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs30:stride2:seed11/config.json @@ -0,0 +1,255 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_uniform_u2_6_stride2_seed11", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 11, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride2-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "uniform", + "uniform_u2_6", + "stride2", + "seed11" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 30.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 11, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride2_seed11/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_observation_stride_uniform_u2_6_stride2_seed11 --train_dir results/checkpoints_factor_sweeps/demon_attack/observation_stride --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 11 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride2_seed11/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 11 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-stride2-uniform_u2_6 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack observation_stride uniform uniform_u2_6 stride2 seed11 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 30.0 --screen_size 84 --noop_max 30 --frame_stack 4 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_uniform_u2_6_stride2_seed11", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 11, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride2-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "uniform", + "uniform_u2_6", + "stride2", + "seed11" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 30.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 11, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride2_seed11/episode_metrics.jsonl" + }, + "git_hash": "a81783762c1f965cc4a9d2eb7f028b47ab17a859", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/observation_stride/train/observation_stride/uniform_u2_6/stride2/seed_11", + "wandb_unique_id": "demon-attack-stride2-uniform_u2_6-s11" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs30:stride2:seed12/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs30:stride2:seed12/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..43fdbb25f83186bc2f722e5e5cc635c8da818762 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs30:stride2:seed12/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a008fb92388716053bf9b6598c91f959050adf12591c44359a6f2394a5d907e0 +size 7210813 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs30:stride2:seed12/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs30:stride2:seed12/config.json new file mode 100644 index 0000000000000000000000000000000000000000..d4b31f3f33f7f7049f55881ae20006a1f83e517f --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs30:stride2:seed12/config.json @@ -0,0 +1,255 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_uniform_u2_6_stride2_seed12", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 12, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride2-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "uniform", + "uniform_u2_6", + "stride2", + "seed12" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 30.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 12, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride2_seed12/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_observation_stride_uniform_u2_6_stride2_seed12 --train_dir results/checkpoints_factor_sweeps/demon_attack/observation_stride --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 12 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride2_seed12/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 12 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-stride2-uniform_u2_6 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack observation_stride uniform uniform_u2_6 stride2 seed12 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 30.0 --screen_size 84 --noop_max 30 --frame_stack 4 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_uniform_u2_6_stride2_seed12", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 12, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride2-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "uniform", + "uniform_u2_6", + "stride2", + "seed12" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 30.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 12, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride2_seed12/episode_metrics.jsonl" + }, + "git_hash": "a81783762c1f965cc4a9d2eb7f028b47ab17a859", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/observation_stride/train/observation_stride/uniform_u2_6/stride2/seed_12", + "wandb_unique_id": "demon-attack-stride2-uniform_u2_6-s12" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs30:stride2:seed13/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs30:stride2:seed13/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..3a4a9aa6d8ef4711d102c1dc442130f03fbf2409 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs30:stride2:seed13/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:45e9362691aea1a3e42d4b2e85766034eecd351ecbd7a4bba1d31582e50261b4 +size 7210813 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs30:stride2:seed13/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs30:stride2:seed13/config.json new file mode 100644 index 0000000000000000000000000000000000000000..787c41573474d1003fa2875cb581e918d847bdc5 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs30:stride2:seed13/config.json @@ -0,0 +1,255 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_uniform_u2_6_stride2_seed13", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 13, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride2-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "uniform", + "uniform_u2_6", + "stride2", + "seed13" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 30.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 13, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride2_seed13/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_observation_stride_uniform_u2_6_stride2_seed13 --train_dir results/checkpoints_factor_sweeps/demon_attack/observation_stride --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 13 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride2_seed13/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 13 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-stride2-uniform_u2_6 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack observation_stride uniform uniform_u2_6 stride2 seed13 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 30.0 --screen_size 84 --noop_max 30 --frame_stack 4 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_uniform_u2_6_stride2_seed13", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 13, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride2-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "uniform", + "uniform_u2_6", + "stride2", + "seed13" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 30.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 13, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride2_seed13/episode_metrics.jsonl" + }, + "git_hash": "a81783762c1f965cc4a9d2eb7f028b47ab17a859", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/observation_stride/train/observation_stride/uniform_u2_6/stride2/seed_13", + "wandb_unique_id": "demon-attack-stride2-uniform_u2_6-s13" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs30:stride2:seed14/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs30:stride2:seed14/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..96009bd803a6dae3d87969ff1b0241474cc53a7d --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs30:stride2:seed14/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aff68a98094d6b572a23e0eb2f35de9e58f2dc8307970e15f69414a650b2c763 +size 7210813 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs30:stride2:seed14/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs30:stride2:seed14/config.json new file mode 100644 index 0000000000000000000000000000000000000000..7950046e7f8dda6adefd3f804aa8becb72b6067c --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs30:stride2:seed14/config.json @@ -0,0 +1,255 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_uniform_u2_6_stride2_seed14", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 14, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride2-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "uniform", + "uniform_u2_6", + "stride2", + "seed14" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 30.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 14, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride2_seed14/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_observation_stride_uniform_u2_6_stride2_seed14 --train_dir results/checkpoints_factor_sweeps/demon_attack/observation_stride --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 14 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride2_seed14/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 14 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-stride2-uniform_u2_6 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack observation_stride uniform uniform_u2_6 stride2 seed14 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 30.0 --screen_size 84 --noop_max 30 --frame_stack 4 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_uniform_u2_6_stride2_seed14", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 14, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride2-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "uniform", + "uniform_u2_6", + "stride2", + "seed14" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 30.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 14, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride2_seed14/episode_metrics.jsonl" + }, + "git_hash": "a81783762c1f965cc4a9d2eb7f028b47ab17a859", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/observation_stride/train/observation_stride/uniform_u2_6/stride2/seed_14", + "wandb_unique_id": "demon-attack-stride2-uniform_u2_6-s14" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs6:stride10:seed10/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs6:stride10:seed10/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..7604444473841631596bb5deff2a9b7da77d66d2 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs6:stride10:seed10/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bb1772474f23b889d18c4411d49d5dc877406e93ee5765b5efcced8541e1ef22 +size 7210813 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs6:stride10:seed10/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs6:stride10:seed10/config.json new file mode 100644 index 0000000000000000000000000000000000000000..f2ab861156b03ec1c25d71fb8b41cdbd116e4807 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs6:stride10:seed10/config.json @@ -0,0 +1,255 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_uniform_u2_6_stride10_seed10", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 10, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride10-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "uniform", + "uniform_u2_6", + "stride10", + "seed10" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 6.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 10, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride10_seed10/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_observation_stride_uniform_u2_6_stride10_seed10 --train_dir results/checkpoints_factor_sweeps/demon_attack/observation_stride --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 10 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride10_seed10/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 10 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 36000 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-stride10-uniform_u2_6 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack observation_stride uniform uniform_u2_6 stride10 seed10 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 6.0 --screen_size 84 --noop_max 30 --frame_stack 4 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_uniform_u2_6_stride10_seed10", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 10, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride10-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "uniform", + "uniform_u2_6", + "stride10", + "seed10" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 6.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 10, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride10_seed10/episode_metrics.jsonl" + }, + "git_hash": "eb3a2e1efbd2aa03a60d7f44f5e18d8fdd0f5a2d", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/observation_stride/train/observation_stride/uniform_u2_6/stride10/seed_10", + "wandb_unique_id": "demon-attack-stride10-uniform_u2_6-s10" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs6:stride10:seed11/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs6:stride10:seed11/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..d6fea103394e9d78aaeb517db8acfb2d1ab7b1b6 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs6:stride10:seed11/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5c679591e2ba4588770f17135f36033195d3bc6e1952d6ce8ae791c54661d97c +size 7210813 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs6:stride10:seed11/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs6:stride10:seed11/config.json new file mode 100644 index 0000000000000000000000000000000000000000..622343c2ecd2a6ea5812cfd1e0bdce04cf15bed1 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs6:stride10:seed11/config.json @@ -0,0 +1,255 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_uniform_u2_6_stride10_seed11", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 11, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride10-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "uniform", + "uniform_u2_6", + "stride10", + "seed11" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 6.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 11, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride10_seed11/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_observation_stride_uniform_u2_6_stride10_seed11 --train_dir results/checkpoints_factor_sweeps/demon_attack/observation_stride --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 11 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride10_seed11/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 11 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 36000 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-stride10-uniform_u2_6 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack observation_stride uniform uniform_u2_6 stride10 seed11 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 6.0 --screen_size 84 --noop_max 30 --frame_stack 4 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_uniform_u2_6_stride10_seed11", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 11, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride10-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "uniform", + "uniform_u2_6", + "stride10", + "seed11" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 6.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 11, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride10_seed11/episode_metrics.jsonl" + }, + "git_hash": "eb3a2e1efbd2aa03a60d7f44f5e18d8fdd0f5a2d", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/observation_stride/train/observation_stride/uniform_u2_6/stride10/seed_11", + "wandb_unique_id": "demon-attack-stride10-uniform_u2_6-s11" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs6:stride10:seed12/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs6:stride10:seed12/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..c37a56c9af29b3c648191f2156240dc0d256e556 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs6:stride10:seed12/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0c3b72c25a47d3a430dbbb491f9c36c87c55a18a5618895cf01d672a0abba23c +size 7210813 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs6:stride10:seed12/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs6:stride10:seed12/config.json new file mode 100644 index 0000000000000000000000000000000000000000..cccdf0c77ba212e3e226a38bc8154d1639858e86 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs6:stride10:seed12/config.json @@ -0,0 +1,255 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_uniform_u2_6_stride10_seed12", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 12, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride10-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "uniform", + "uniform_u2_6", + "stride10", + "seed12" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 6.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 12, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride10_seed12/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_observation_stride_uniform_u2_6_stride10_seed12 --train_dir results/checkpoints_factor_sweeps/demon_attack/observation_stride --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 12 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride10_seed12/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 12 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 36000 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-stride10-uniform_u2_6 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack observation_stride uniform uniform_u2_6 stride10 seed12 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 6.0 --screen_size 84 --noop_max 30 --frame_stack 4 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_uniform_u2_6_stride10_seed12", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 12, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride10-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "uniform", + "uniform_u2_6", + "stride10", + "seed12" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 6.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 12, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride10_seed12/episode_metrics.jsonl" + }, + "git_hash": "eb3a2e1efbd2aa03a60d7f44f5e18d8fdd0f5a2d", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/observation_stride/train/observation_stride/uniform_u2_6/stride10/seed_12", + "wandb_unique_id": "demon-attack-stride10-uniform_u2_6-s12" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs6:stride10:seed13/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs6:stride10:seed13/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..9f646c23196e0b3ac73a68dcd666d19b797d2e88 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs6:stride10:seed13/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c47c934775721a8fcb12c21caffa5e9d34018203682cbb56da835bc716c45ef5 +size 7210813 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs6:stride10:seed13/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs6:stride10:seed13/config.json new file mode 100644 index 0000000000000000000000000000000000000000..eaa1a3e1f34c4291dad15c1c04fb3861a3614d19 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs6:stride10:seed13/config.json @@ -0,0 +1,255 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_uniform_u2_6_stride10_seed13", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 13, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride10-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "uniform", + "uniform_u2_6", + "stride10", + "seed13" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 6.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 13, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride10_seed13/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_observation_stride_uniform_u2_6_stride10_seed13 --train_dir results/checkpoints_factor_sweeps/demon_attack/observation_stride --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 13 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride10_seed13/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 13 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 36000 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-stride10-uniform_u2_6 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack observation_stride uniform uniform_u2_6 stride10 seed13 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 6.0 --screen_size 84 --noop_max 30 --frame_stack 4 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_uniform_u2_6_stride10_seed13", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 13, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride10-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "uniform", + "uniform_u2_6", + "stride10", + "seed13" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 6.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 13, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride10_seed13/episode_metrics.jsonl" + }, + "git_hash": "eb3a2e1efbd2aa03a60d7f44f5e18d8fdd0f5a2d", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/observation_stride/train/observation_stride/uniform_u2_6/stride10/seed_13", + "wandb_unique_id": "demon-attack-stride10-uniform_u2_6-s13" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs6:stride10:seed14/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs6:stride10:seed14/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..ce9cc95230bfcbd7562808d8ffa3d44bdd31377d --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs6:stride10:seed14/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:463219314336aecc9b00b480aa835319e531429638a2ee37733d718461359b6d +size 7210813 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs6:stride10:seed14/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs6:stride10:seed14/config.json new file mode 100644 index 0000000000000000000000000000000000000000..2509a233b3161c5ea120772bfca17aeb77b17107 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs6:stride10:seed14/config.json @@ -0,0 +1,255 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_uniform_u2_6_stride10_seed14", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 14, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride10-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "uniform", + "uniform_u2_6", + "stride10", + "seed14" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 6.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 14, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride10_seed14/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_observation_stride_uniform_u2_6_stride10_seed14 --train_dir results/checkpoints_factor_sweeps/demon_attack/observation_stride --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 14 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride10_seed14/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 14 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 36000 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-stride10-uniform_u2_6 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack observation_stride uniform uniform_u2_6 stride10 seed14 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 6.0 --screen_size 84 --noop_max 30 --frame_stack 4 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_uniform_u2_6_stride10_seed14", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 14, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride10-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "uniform", + "uniform_u2_6", + "stride10", + "seed14" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 6.0, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 14, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride10_seed14/episode_metrics.jsonl" + }, + "git_hash": "eb3a2e1efbd2aa03a60d7f44f5e18d8fdd0f5a2d", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/observation_stride/train/observation_stride/uniform_u2_6/stride10/seed_14", + "wandb_unique_id": "demon-attack-stride10-uniform_u2_6-s14" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs6p66667:stride9:seed10/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs6p66667:stride9:seed10/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..712c20db77a5c089f646cff3a680ddcf9ece4faf --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs6p66667:stride9:seed10/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d75978b25c67120891f42161593e266267737304e7c6a64de019f1ef871b6084 +size 7210813 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs6p66667:stride9:seed10/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs6p66667:stride9:seed10/config.json new file mode 100644 index 0000000000000000000000000000000000000000..bad3bcd5a3364914c3a6977b22529af9730fb506 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs6p66667:stride9:seed10/config.json @@ -0,0 +1,255 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_uniform_u2_6_stride9_seed10", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 10, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride9-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "uniform", + "uniform_u2_6", + "stride9", + "seed10" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 6.666666666666667, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 10, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride9_seed10/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_observation_stride_uniform_u2_6_stride9_seed10 --train_dir results/checkpoints_factor_sweeps/demon_attack/observation_stride --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 10 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride9_seed10/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 10 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 36000 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-stride9-uniform_u2_6 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack observation_stride uniform uniform_u2_6 stride9 seed10 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 6.666666666666667 --screen_size 84 --noop_max 30 --frame_stack 4 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_uniform_u2_6_stride9_seed10", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 10, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride9-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "uniform", + "uniform_u2_6", + "stride9", + "seed10" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 6.666666666666667, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 10, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride9_seed10/episode_metrics.jsonl" + }, + "git_hash": "eb3a2e1efbd2aa03a60d7f44f5e18d8fdd0f5a2d", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/observation_stride/train/observation_stride/uniform_u2_6/stride9/seed_10", + "wandb_unique_id": "demon-attack-stride9-uniform_u2_6-s10" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs6p66667:stride9:seed11/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs6p66667:stride9:seed11/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..f9de443619f627fdd26d31913ada675278e175ae --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs6p66667:stride9:seed11/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4f49229d108ebdfd944e5805d23b6819322f751c7334eddd274b59cee9e9ed21 +size 7210813 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs6p66667:stride9:seed11/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs6p66667:stride9:seed11/config.json new file mode 100644 index 0000000000000000000000000000000000000000..4a2430848c249bb7a36f1347989cb7d0005791a7 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs6p66667:stride9:seed11/config.json @@ -0,0 +1,255 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_uniform_u2_6_stride9_seed11", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 11, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride9-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "uniform", + "uniform_u2_6", + "stride9", + "seed11" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 6.666666666666667, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 11, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride9_seed11/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_observation_stride_uniform_u2_6_stride9_seed11 --train_dir results/checkpoints_factor_sweeps/demon_attack/observation_stride --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 11 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride9_seed11/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 11 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 36000 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-stride9-uniform_u2_6 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack observation_stride uniform uniform_u2_6 stride9 seed11 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 6.666666666666667 --screen_size 84 --noop_max 30 --frame_stack 4 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_uniform_u2_6_stride9_seed11", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 11, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride9-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "uniform", + "uniform_u2_6", + "stride9", + "seed11" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 6.666666666666667, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 11, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride9_seed11/episode_metrics.jsonl" + }, + "git_hash": "eb3a2e1efbd2aa03a60d7f44f5e18d8fdd0f5a2d", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/observation_stride/train/observation_stride/uniform_u2_6/stride9/seed_11", + "wandb_unique_id": "demon-attack-stride9-uniform_u2_6-s11" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs6p66667:stride9:seed12/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs6p66667:stride9:seed12/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..32a9decd0ed091b5c6f46215e8901d198611b44f --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs6p66667:stride9:seed12/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d71bc2bcab4d24a1cc76eb69a97e9fa3fef6c3509d61c8ef00e7cd4456a58cd1 +size 7210813 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs6p66667:stride9:seed12/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs6p66667:stride9:seed12/config.json new file mode 100644 index 0000000000000000000000000000000000000000..260c9e5a2bcd2acb2cb0cccdcc3e0470e6dcf66a --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs6p66667:stride9:seed12/config.json @@ -0,0 +1,255 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_uniform_u2_6_stride9_seed12", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 12, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride9-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "uniform", + "uniform_u2_6", + "stride9", + "seed12" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 6.666666666666667, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 12, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride9_seed12/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_observation_stride_uniform_u2_6_stride9_seed12 --train_dir results/checkpoints_factor_sweeps/demon_attack/observation_stride --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 12 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride9_seed12/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 12 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 36000 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-stride9-uniform_u2_6 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack observation_stride uniform uniform_u2_6 stride9 seed12 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 6.666666666666667 --screen_size 84 --noop_max 30 --frame_stack 4 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_uniform_u2_6_stride9_seed12", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 12, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride9-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "uniform", + "uniform_u2_6", + "stride9", + "seed12" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 6.666666666666667, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 12, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride9_seed12/episode_metrics.jsonl" + }, + "git_hash": "eb3a2e1efbd2aa03a60d7f44f5e18d8fdd0f5a2d", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/observation_stride/train/observation_stride/uniform_u2_6/stride9/seed_12", + "wandb_unique_id": "demon-attack-stride9-uniform_u2_6-s12" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs6p66667:stride9:seed13/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs6p66667:stride9:seed13/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..c1f00d7aae1f0de45eec1d1066feca83d4165432 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs6p66667:stride9:seed13/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8092e383598be856c966934bc1402823306d588ffe8d67a1948446e012a6690f +size 7210813 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs6p66667:stride9:seed13/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs6p66667:stride9:seed13/config.json new file mode 100644 index 0000000000000000000000000000000000000000..a37a0acf4dd696a96f99068eae1d4e982e17b3a1 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs6p66667:stride9:seed13/config.json @@ -0,0 +1,255 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_uniform_u2_6_stride9_seed13", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 13, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride9-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "uniform", + "uniform_u2_6", + "stride9", + "seed13" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 6.666666666666667, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 13, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride9_seed13/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_observation_stride_uniform_u2_6_stride9_seed13 --train_dir results/checkpoints_factor_sweeps/demon_attack/observation_stride --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 13 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride9_seed13/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 13 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 36000 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-stride9-uniform_u2_6 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack observation_stride uniform uniform_u2_6 stride9 seed13 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 6.666666666666667 --screen_size 84 --noop_max 30 --frame_stack 4 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_uniform_u2_6_stride9_seed13", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 13, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride9-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "uniform", + "uniform_u2_6", + "stride9", + "seed13" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 6.666666666666667, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 13, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride9_seed13/episode_metrics.jsonl" + }, + "git_hash": "eb3a2e1efbd2aa03a60d7f44f5e18d8fdd0f5a2d", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/observation_stride/train/observation_stride/uniform_u2_6/stride9/seed_13", + "wandb_unique_id": "demon-attack-stride9-uniform_u2_6-s13" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs6p66667:stride9:seed14/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs6p66667:stride9:seed14/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..2e8a729ed603e2d6323508c58f68c93647a58aea --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs6p66667:stride9:seed14/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8a29b75fdd6c1b887fadf9780ec9fe96ccb854814967b674765c67beb4e2d4c4 +size 7210813 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs6p66667:stride9:seed14/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs6p66667:stride9:seed14/config.json new file mode 100644 index 0000000000000000000000000000000000000000..19f86b54605d16ecdecd972b896495ddab415d48 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs6p66667:stride9:seed14/config.json @@ -0,0 +1,255 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_uniform_u2_6_stride9_seed14", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 14, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride9-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "uniform", + "uniform_u2_6", + "stride9", + "seed14" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 6.666666666666667, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 14, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride9_seed14/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_observation_stride_uniform_u2_6_stride9_seed14 --train_dir results/checkpoints_factor_sweeps/demon_attack/observation_stride --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 14 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride9_seed14/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 14 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 36000 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-stride9-uniform_u2_6 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack observation_stride uniform uniform_u2_6 stride9 seed14 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 6.666666666666667 --screen_size 84 --noop_max 30 --frame_stack 4 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_uniform_u2_6_stride9_seed14", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 14, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride9-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "uniform", + "uniform_u2_6", + "stride9", + "seed14" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 6.666666666666667, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 14, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride9_seed14/episode_metrics.jsonl" + }, + "git_hash": "eb3a2e1efbd2aa03a60d7f44f5e18d8fdd0f5a2d", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/observation_stride/train/observation_stride/uniform_u2_6/stride9/seed_14", + "wandb_unique_id": "demon-attack-stride9-uniform_u2_6-s14" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs7p5:stride8:seed10/checkpoint_p0/checkpoint_000048928_25051136.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs7p5:stride8:seed10/checkpoint_p0/checkpoint_000048928_25051136.pth new file mode 100644 index 0000000000000000000000000000000000000000..b99f9a36557545cd7b5d9469c1e4838c43eb271f --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs7p5:stride8:seed10/checkpoint_p0/checkpoint_000048928_25051136.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f35e2e9830793a749e5a2a8a22726d996f8d54504cfc3e652802389c5e1978c7 +size 7210813 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs7p5:stride8:seed10/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs7p5:stride8:seed10/config.json new file mode 100644 index 0000000000000000000000000000000000000000..4f1078e8318fe4023625de5fbf6e23d6fcaad5eb --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs7p5:stride8:seed10/config.json @@ -0,0 +1,255 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_uniform_u2_6_stride8_seed10", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 10, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride8-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "uniform", + "uniform_u2_6", + "stride8", + "seed10" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 7.5, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 10, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride8_seed10/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_observation_stride_uniform_u2_6_stride8_seed10 --train_dir results/checkpoints_factor_sweeps/demon_attack/observation_stride --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 10 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride8_seed10/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 10 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 36000 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-stride8-uniform_u2_6 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack observation_stride uniform uniform_u2_6 stride8 seed10 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 7.5 --screen_size 84 --noop_max 30 --frame_stack 4 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_uniform_u2_6_stride8_seed10", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 10, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride8-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "uniform", + "uniform_u2_6", + "stride8", + "seed10" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 7.5, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 10, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride8_seed10/episode_metrics.jsonl" + }, + "git_hash": "eb3a2e1efbd2aa03a60d7f44f5e18d8fdd0f5a2d", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/observation_stride/train/observation_stride/uniform_u2_6/stride8/seed_10", + "wandb_unique_id": "demon-attack-stride8-uniform_u2_6-s10" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs7p5:stride8:seed11/checkpoint_p0/checkpoint_000048928_25051136.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs7p5:stride8:seed11/checkpoint_p0/checkpoint_000048928_25051136.pth new file mode 100644 index 0000000000000000000000000000000000000000..6f9fd8d3df50aab1f8fb0be73014ce2ade7680e9 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs7p5:stride8:seed11/checkpoint_p0/checkpoint_000048928_25051136.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:416180bf8ce49e6ef921f9d83080e62e325c9cf77689dfdfab6ec9e9413a069c +size 7210813 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs7p5:stride8:seed11/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs7p5:stride8:seed11/config.json new file mode 100644 index 0000000000000000000000000000000000000000..2ae6d9616c90f2138df0f7761ecd706a7451e81c --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs7p5:stride8:seed11/config.json @@ -0,0 +1,255 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_uniform_u2_6_stride8_seed11", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 11, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride8-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "uniform", + "uniform_u2_6", + "stride8", + "seed11" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 7.5, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 11, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride8_seed11/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_observation_stride_uniform_u2_6_stride8_seed11 --train_dir results/checkpoints_factor_sweeps/demon_attack/observation_stride --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 11 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride8_seed11/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 11 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 36000 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-stride8-uniform_u2_6 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack observation_stride uniform uniform_u2_6 stride8 seed11 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 7.5 --screen_size 84 --noop_max 30 --frame_stack 4 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_uniform_u2_6_stride8_seed11", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 11, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride8-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "uniform", + "uniform_u2_6", + "stride8", + "seed11" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 7.5, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 11, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride8_seed11/episode_metrics.jsonl" + }, + "git_hash": "eb3a2e1efbd2aa03a60d7f44f5e18d8fdd0f5a2d", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/observation_stride/train/observation_stride/uniform_u2_6/stride8/seed_11", + "wandb_unique_id": "demon-attack-stride8-uniform_u2_6-s11" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs7p5:stride8:seed12/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs7p5:stride8:seed12/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..c94490cf856f716a578ba7a1c4a291d12d029d3f --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs7p5:stride8:seed12/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2b9d929b66cd0ef33537b9dd298a77e7a91f396e27ccce647fdbc7d3184e704d +size 7210813 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs7p5:stride8:seed12/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs7p5:stride8:seed12/config.json new file mode 100644 index 0000000000000000000000000000000000000000..c23fdc02d4069aa88fff1a9b9aff95458be4549a --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs7p5:stride8:seed12/config.json @@ -0,0 +1,255 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_uniform_u2_6_stride8_seed12", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 12, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride8-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "uniform", + "uniform_u2_6", + "stride8", + "seed12" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 7.5, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 12, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride8_seed12/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_observation_stride_uniform_u2_6_stride8_seed12 --train_dir results/checkpoints_factor_sweeps/demon_attack/observation_stride --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 12 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride8_seed12/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 12 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 36000 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-stride8-uniform_u2_6 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack observation_stride uniform uniform_u2_6 stride8 seed12 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 7.5 --screen_size 84 --noop_max 30 --frame_stack 4 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_uniform_u2_6_stride8_seed12", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 12, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride8-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "uniform", + "uniform_u2_6", + "stride8", + "seed12" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 7.5, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 12, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride8_seed12/episode_metrics.jsonl" + }, + "git_hash": "eb3a2e1efbd2aa03a60d7f44f5e18d8fdd0f5a2d", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/observation_stride/train/observation_stride/uniform_u2_6/stride8/seed_12", + "wandb_unique_id": "demon-attack-stride8-uniform_u2_6-s12" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs7p5:stride8:seed13/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs7p5:stride8:seed13/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..e8a1818aaf5807d67a6ebbd1e230981b600caced --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs7p5:stride8:seed13/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7826bf7665fe8338166a167d912ede78daf3c4a56fe165a124666a7aa4ad1da7 +size 7210813 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs7p5:stride8:seed13/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs7p5:stride8:seed13/config.json new file mode 100644 index 0000000000000000000000000000000000000000..c9cca9059ee096933cdea18d2045bc3cc4a2ae7c --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs7p5:stride8:seed13/config.json @@ -0,0 +1,255 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_uniform_u2_6_stride8_seed13", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 13, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride8-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "uniform", + "uniform_u2_6", + "stride8", + "seed13" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 7.5, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 13, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride8_seed13/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_observation_stride_uniform_u2_6_stride8_seed13 --train_dir results/checkpoints_factor_sweeps/demon_attack/observation_stride --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 13 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride8_seed13/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 13 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 36000 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-stride8-uniform_u2_6 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack observation_stride uniform uniform_u2_6 stride8 seed13 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 7.5 --screen_size 84 --noop_max 30 --frame_stack 4 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_uniform_u2_6_stride8_seed13", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 13, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride8-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "uniform", + "uniform_u2_6", + "stride8", + "seed13" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 7.5, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 13, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride8_seed13/episode_metrics.jsonl" + }, + "git_hash": "eb3a2e1efbd2aa03a60d7f44f5e18d8fdd0f5a2d", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/observation_stride/train/observation_stride/uniform_u2_6/stride8/seed_13", + "wandb_unique_id": "demon-attack-stride8-uniform_u2_6-s13" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs7p5:stride8:seed14/checkpoint_p0/checkpoint_000048928_25051136.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs7p5:stride8:seed14/checkpoint_p0/checkpoint_000048928_25051136.pth new file mode 100644 index 0000000000000000000000000000000000000000..8446f2b694ef6d67006b543c15d1e02734e56117 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs7p5:stride8:seed14/checkpoint_p0/checkpoint_000048928_25051136.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:03ee55173243cd33754017a5ed79c283445d954fe13852c269c8286b1a9aa69d +size 7210813 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs7p5:stride8:seed14/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs7p5:stride8:seed14/config.json new file mode 100644 index 0000000000000000000000000000000000000000..083348205cb4449394bc0ebe1abb7983eacc3348 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs7p5:stride8:seed14/config.json @@ -0,0 +1,255 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_uniform_u2_6_stride8_seed14", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 14, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride8-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "uniform", + "uniform_u2_6", + "stride8", + "seed14" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 7.5, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 14, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride8_seed14/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_observation_stride_uniform_u2_6_stride8_seed14 --train_dir results/checkpoints_factor_sweeps/demon_attack/observation_stride --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 14 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride8_seed14/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 14 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 36000 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-stride8-uniform_u2_6 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack observation_stride uniform uniform_u2_6 stride8 seed14 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 7.5 --screen_size 84 --noop_max 30 --frame_stack 4 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_uniform_u2_6_stride8_seed14", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 14, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride8-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "uniform", + "uniform_u2_6", + "stride8", + "seed14" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 7.5, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 14, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride8_seed14/episode_metrics.jsonl" + }, + "git_hash": "eb3a2e1efbd2aa03a60d7f44f5e18d8fdd0f5a2d", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/observation_stride/train/observation_stride/uniform_u2_6/stride8/seed_14", + "wandb_unique_id": "demon-attack-stride8-uniform_u2_6-s14" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs8p57143:stride7:seed10/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs8p57143:stride7:seed10/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..84b3cd69cdcec3199452dc6e2fd15bceaf236e36 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs8p57143:stride7:seed10/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f42a8a7894dbffcd6d6a27b349ce8e0e5980205c65b3fda0f42cad7bf3a4a14e +size 7210813 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs8p57143:stride7:seed10/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs8p57143:stride7:seed10/config.json new file mode 100644 index 0000000000000000000000000000000000000000..a954c8f30f3f7ea1ebae6405e1dbfad862ea3d9f --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs8p57143:stride7:seed10/config.json @@ -0,0 +1,255 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_uniform_u2_6_stride7_seed10", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 10, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride7-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "uniform", + "uniform_u2_6", + "stride7", + "seed10" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 8.571428571428571, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 10, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride7_seed10/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_observation_stride_uniform_u2_6_stride7_seed10 --train_dir results/checkpoints_factor_sweeps/demon_attack/observation_stride --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 10 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride7_seed10/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 10 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 36000 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-stride7-uniform_u2_6 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack observation_stride uniform uniform_u2_6 stride7 seed10 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 8.571428571428571 --screen_size 84 --noop_max 30 --frame_stack 4 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_uniform_u2_6_stride7_seed10", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 10, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride7-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "uniform", + "uniform_u2_6", + "stride7", + "seed10" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 8.571428571428571, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 10, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride7_seed10/episode_metrics.jsonl" + }, + "git_hash": "eb3a2e1efbd2aa03a60d7f44f5e18d8fdd0f5a2d", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/observation_stride/train/observation_stride/uniform_u2_6/stride7/seed_10", + "wandb_unique_id": "demon-attack-stride7-uniform_u2_6-s10" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs8p57143:stride7:seed11/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs8p57143:stride7:seed11/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..6728c48ef9ada373f748eed32d61f90ecc0827a8 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs8p57143:stride7:seed11/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e717cccdd9b93be8ba483e2a367d2b12379c984857e272b0ef73b9852d780eff +size 7210813 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs8p57143:stride7:seed11/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs8p57143:stride7:seed11/config.json new file mode 100644 index 0000000000000000000000000000000000000000..3ccc67732e3cb878302eb979360fddb227933510 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs8p57143:stride7:seed11/config.json @@ -0,0 +1,255 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_uniform_u2_6_stride7_seed11", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 11, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride7-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "uniform", + "uniform_u2_6", + "stride7", + "seed11" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 8.571428571428571, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 11, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride7_seed11/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_observation_stride_uniform_u2_6_stride7_seed11 --train_dir results/checkpoints_factor_sweeps/demon_attack/observation_stride --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 11 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride7_seed11/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 11 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 36000 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-stride7-uniform_u2_6 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack observation_stride uniform uniform_u2_6 stride7 seed11 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 8.571428571428571 --screen_size 84 --noop_max 30 --frame_stack 4 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_uniform_u2_6_stride7_seed11", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 11, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride7-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "uniform", + "uniform_u2_6", + "stride7", + "seed11" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 8.571428571428571, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 11, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride7_seed11/episode_metrics.jsonl" + }, + "git_hash": "eb3a2e1efbd2aa03a60d7f44f5e18d8fdd0f5a2d", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/observation_stride/train/observation_stride/uniform_u2_6/stride7/seed_11", + "wandb_unique_id": "demon-attack-stride7-uniform_u2_6-s11" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs8p57143:stride7:seed12/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs8p57143:stride7:seed12/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..4292b9885ee97c530dc67c7c7bc868b3890c9f6f --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs8p57143:stride7:seed12/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5a8a5350a22d442553f919a63310d7c503afbac8415673a7fa7f99211a9601f5 +size 7210813 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs8p57143:stride7:seed12/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs8p57143:stride7:seed12/config.json new file mode 100644 index 0000000000000000000000000000000000000000..b95d7eb826412b82e0db57fa0625e53e63031989 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs8p57143:stride7:seed12/config.json @@ -0,0 +1,255 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_uniform_u2_6_stride7_seed12", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 12, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride7-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "uniform", + "uniform_u2_6", + "stride7", + "seed12" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 8.571428571428571, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 12, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride7_seed12/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_observation_stride_uniform_u2_6_stride7_seed12 --train_dir results/checkpoints_factor_sweeps/demon_attack/observation_stride --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 12 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride7_seed12/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 12 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 36000 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-stride7-uniform_u2_6 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack observation_stride uniform uniform_u2_6 stride7 seed12 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 8.571428571428571 --screen_size 84 --noop_max 30 --frame_stack 4 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_uniform_u2_6_stride7_seed12", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 12, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride7-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "uniform", + "uniform_u2_6", + "stride7", + "seed12" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 8.571428571428571, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 12, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride7_seed12/episode_metrics.jsonl" + }, + "git_hash": "eb3a2e1efbd2aa03a60d7f44f5e18d8fdd0f5a2d", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/observation_stride/train/observation_stride/uniform_u2_6/stride7/seed_12", + "wandb_unique_id": "demon-attack-stride7-uniform_u2_6-s12" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs8p57143:stride7:seed13/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs8p57143:stride7:seed13/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..fb34e1048747161e1c2cae5d112cdf0e70788a8e --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs8p57143:stride7:seed13/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4def1deb818617320f46ffcb47969965a27b21379f6c13e9ea3f3165742e22d9 +size 7210813 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs8p57143:stride7:seed13/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs8p57143:stride7:seed13/config.json new file mode 100644 index 0000000000000000000000000000000000000000..e567dc5ec2ee338afacbfcd6e0cdc821096c281e --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs8p57143:stride7:seed13/config.json @@ -0,0 +1,255 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_uniform_u2_6_stride7_seed13", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 13, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride7-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "uniform", + "uniform_u2_6", + "stride7", + "seed13" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 8.571428571428571, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 13, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride7_seed13/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_observation_stride_uniform_u2_6_stride7_seed13 --train_dir results/checkpoints_factor_sweeps/demon_attack/observation_stride --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 13 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride7_seed13/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 13 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 36000 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-stride7-uniform_u2_6 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack observation_stride uniform uniform_u2_6 stride7 seed13 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 8.571428571428571 --screen_size 84 --noop_max 30 --frame_stack 4 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_uniform_u2_6_stride7_seed13", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 13, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride7-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "uniform", + "uniform_u2_6", + "stride7", + "seed13" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 8.571428571428571, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 13, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride7_seed13/episode_metrics.jsonl" + }, + "git_hash": "eb3a2e1efbd2aa03a60d7f44f5e18d8fdd0f5a2d", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/observation_stride/train/observation_stride/uniform_u2_6/stride7/seed_13", + "wandb_unique_id": "demon-attack-stride7-uniform_u2_6-s13" +} \ No newline at end of file diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs8p57143:stride7:seed14/checkpoint_p0/checkpoint_000048880_25026560.pth b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs8p57143:stride7:seed14/checkpoint_p0/checkpoint_000048880_25026560.pth new file mode 100644 index 0000000000000000000000000000000000000000..9d043aba6a75abe39c68dc9dc2ea5bcb19d38b67 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs8p57143:stride7:seed14/checkpoint_p0/checkpoint_000048880_25026560.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7a0259f954e68712bcb44e4c1fd17fc8e19181cda595bd55924018b3a6784388 +size 7210813 diff --git a/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs8p57143:stride7:seed14/config.json b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs8p57143:stride7:seed14/config.json new file mode 100644 index 0000000000000000000000000000000000000000..63f1b877deecfb061297b00835b4dc3fbac78453 --- /dev/null +++ b/latency-aware/demon-attack/small-policy/factor_sweep:demon_attack:observation_stride:uniform_u2_6:fs4:obs8p57143:stride7:seed14/config.json @@ -0,0 +1,255 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_uniform_u2_6_stride7_seed14", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 14, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride7-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "uniform", + "uniform_u2_6", + "stride7", + "seed14" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 8.571428571428571, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "export_env_raw_rgb_frames": false, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 14, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride7_seed14/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_demon_attack --experiment demon_attack_observation_stride_uniform_u2_6_stride7_seed14 --train_dir results/checkpoints_factor_sweeps/demon_attack/observation_stride --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-env-batch-size 512 --seed 14 --episode_metrics_path results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride7_seed14/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 1 --num_envs_per_worker 1 --num_policies 1 --batch_size 1024 --rollout 64 --recurrence 32 --num_epochs 2 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.000303 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.0 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --batched_sampling True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 14 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 36000 --eval-deterministic True --eval-raw-reward --with_wandb True --wandb_project latency-sensitive-bench --wandb_group demon-attack-stride7-uniform_u2_6 --wandb_job_type sample_factory --wandb_tags factor_sweep demon-attack observation_stride uniform uniform_u2_6 stride7 seed14 --env_id ALE/DemonAttack-v5 --env-fps 60 --obs-fps 8.571428571428571 --screen_size 84 --noop_max 30 --frame_stack 4 --terminal_on_life_loss True --clip_reward True --repeat_action_probability 0.0 --full_action_space False --max_episode_steps 25000 --simulator gpu --gpu-env-device auto --hold-policy hold --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_demon_attack", + "experiment": "demon_attack_observation_stride_uniform_u2_6_stride7_seed14", + "train_dir": "results/checkpoints_factor_sweeps/demon_attack/observation_stride", + "restart_behavior": "resume", + "device": "gpu", + "seed": 14, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 300, + "num_workers": 1, + "num_envs_per_worker": 1, + "batch_size": 1024, + "num_batches_per_epoch": 8, + "num_epochs": 2, + "rollout": 64, + "recurrence": 32, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.0, + "learning_rate": 0.000303, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "demon-attack-stride7-uniform_u2_6", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "demon-attack", + "observation_stride", + "uniform", + "uniform_u2_6", + "stride7", + "seed14" + ], + "env_id": "ALE/DemonAttack-v5", + "env_fps": 60.0, + "obs_fps": 8.571428571428571, + "screen_size": 84, + "noop_max": 30, + "frame_stack": 4, + "terminal_on_life_loss": true, + "clip_reward": true, + "repeat_action_probability": 0.0, + "full_action_space": false, + "max_episode_steps": 25000, + "simulator": "gpu", + "gpu_env_device": "auto", + "gpu_env_batch_size": 512, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 14, + "add_latency_info": false, + "hold_policy": "hold", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 36000, + "eval_deterministic": true, + "eval_raw_reward": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/demon_attack/observation_stride/demon_attack_observation_stride_uniform_u2_6_stride7_seed14/episode_metrics.jsonl" + }, + "git_hash": "eb3a2e1efbd2aa03a60d7f44f5e18d8fdd0f5a2d", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/demon_attack/observation_stride/train/observation_stride/uniform_u2_6/stride7/seed_14", + "wandb_unique_id": "demon-attack-stride7-uniform_u2_6-s14" +} \ No newline at end of file diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs1:obs30:stride1:seed10/checkpoint_p0/checkpoint_000048848_25034752.pth b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs1:obs30:stride1:seed10/checkpoint_p0/checkpoint_000048848_25034752.pth new file mode 100644 index 0000000000000000000000000000000000000000..35d9cf79e4860f5948742558ba5b8c7f48901c17 --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs1:obs30:stride1:seed10/checkpoint_p0/checkpoint_000048848_25034752.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7c3ddd4a24a135d19ebcc8845fda00b9f743b8e3ff4c741aa05be800ce431fd7 +size 7081533 diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs1:obs30:stride1:seed10/config.json b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs1:obs30:stride1:seed10/config.json new file mode 100644 index 0000000000000000000000000000000000000000..1f885ed7aa158c3aaa176e13761f0b598c6b6365 --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs1:obs30:stride1:seed10/config.json @@ -0,0 +1,266 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_fixed_l2_fs1_seed10", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 10, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs1-fixed_l2", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "fixed", + "fixed_l2", + "fs1", + "seed10" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "screen_size": "", + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "score_limit": -1, + "frame_stack": 1, + "debug": false, + "debug_timelimit_diagnostics": false, + "max_episode_steps": 0, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": null, + "max_latency_ms": null, + "latency_seed": null, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": false, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs1_seed10/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_flappy --experiment flappy_frame_stack_fixed_l2_fs1_seed10 --train_dir results/checkpoints_factor_sweeps/flappy/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-render-batch-size 128 --seed 10 --episode_metrics_path results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs1_seed10/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 2 --num_envs_per_worker 1 --num_policies 1 --batch_size 4096 --rollout 128 --recurrence 1 --num_epochs 8 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 400 --learning_rate 0.00025 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.5 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --use_rnn False --normalize_returns True --normalize_input True --latency-type fixed --fixed-latency-ms 66.66666666666667 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --with_wandb True --wandb_project latency-sensitive-bench --wandb_group flappy-fs1-fixed_l2 --wandb_job_type sample_factory --wandb_tags factor_sweep flappy frame_stack fixed fixed_l2 fs1 seed10 --gym_id FlappyBird-v0 --env-fps 30 --obs-fps 30.0 --use_lidar False --normalize_obs True --audio_on False --obs_resize 84,84 --use-gpu-render True --simulator gpu --gpu-render-device auto --gpu-render-profile False --gpu-render-profile-interval 200 --pipe_gap 100 --bird_color yellow --pipe_color green --background day --frame_stack 1 --debug False --debug-timelimit-diagnostics False --hold-policy one_frame_then_noop --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_fixed_l2_fs1_seed10", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 10, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs1-fixed_l2", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "fixed", + "fixed_l2", + "fs1", + "seed10" + ], + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "frame_stack": 1, + "debug": false, + "debug_timelimit_diagnostics": false, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "add_latency_info": false, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs1_seed10/episode_metrics.jsonl" + }, + "git_hash": "eb3a2e1efbd2aa03a60d7f44f5e18d8fdd0f5a2d", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/flappy/context_window/train/frame_stack/fixed_l2/fs1/seed_10", + "wandb_unique_id": "flappy-fs1-fixed_l2-s10" +} \ No newline at end of file diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs1:obs30:stride1:seed11/checkpoint_p0/checkpoint_000048840_25034752.pth b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs1:obs30:stride1:seed11/checkpoint_p0/checkpoint_000048840_25034752.pth new file mode 100644 index 0000000000000000000000000000000000000000..1982db4cb4d4294221a8dbceb4f8cc7d502de3d9 --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs1:obs30:stride1:seed11/checkpoint_p0/checkpoint_000048840_25034752.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e8c2a2ac6b858d8a7236ca1b9a50c59b87ebe01f7e6d53ecbb977794fcced6b2 +size 7081533 diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs1:obs30:stride1:seed11/config.json b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs1:obs30:stride1:seed11/config.json new file mode 100644 index 0000000000000000000000000000000000000000..9459768109d2db4ec55c85681366c50ec332eb2f --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs1:obs30:stride1:seed11/config.json @@ -0,0 +1,266 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_fixed_l2_fs1_seed11", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 11, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs1-fixed_l2", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "fixed", + "fixed_l2", + "fs1", + "seed11" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "screen_size": "", + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "score_limit": -1, + "frame_stack": 1, + "debug": false, + "debug_timelimit_diagnostics": false, + "max_episode_steps": 0, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": null, + "max_latency_ms": null, + "latency_seed": null, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": false, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs1_seed11/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_flappy --experiment flappy_frame_stack_fixed_l2_fs1_seed11 --train_dir results/checkpoints_factor_sweeps/flappy/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-render-batch-size 128 --seed 11 --episode_metrics_path results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs1_seed11/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 2 --num_envs_per_worker 1 --num_policies 1 --batch_size 4096 --rollout 128 --recurrence 1 --num_epochs 8 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 400 --learning_rate 0.00025 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.5 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --use_rnn False --normalize_returns True --normalize_input True --latency-type fixed --fixed-latency-ms 66.66666666666667 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --with_wandb True --wandb_project latency-sensitive-bench --wandb_group flappy-fs1-fixed_l2 --wandb_job_type sample_factory --wandb_tags factor_sweep flappy frame_stack fixed fixed_l2 fs1 seed11 --gym_id FlappyBird-v0 --env-fps 30 --obs-fps 30.0 --use_lidar False --normalize_obs True --audio_on False --obs_resize 84,84 --use-gpu-render True --simulator gpu --gpu-render-device auto --gpu-render-profile False --gpu-render-profile-interval 200 --pipe_gap 100 --bird_color yellow --pipe_color green --background day --frame_stack 1 --debug False --debug-timelimit-diagnostics False --hold-policy one_frame_then_noop --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_fixed_l2_fs1_seed11", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 11, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs1-fixed_l2", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "fixed", + "fixed_l2", + "fs1", + "seed11" + ], + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "frame_stack": 1, + "debug": false, + "debug_timelimit_diagnostics": false, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "add_latency_info": false, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs1_seed11/episode_metrics.jsonl" + }, + "git_hash": "eb3a2e1efbd2aa03a60d7f44f5e18d8fdd0f5a2d", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/flappy/context_window/train/frame_stack/fixed_l2/fs1/seed_11", + "wandb_unique_id": "flappy-fs1-fixed_l2-s11" +} \ No newline at end of file diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs1:obs30:stride1:seed12/checkpoint_p0/checkpoint_000048840_25034752.pth b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs1:obs30:stride1:seed12/checkpoint_p0/checkpoint_000048840_25034752.pth new file mode 100644 index 0000000000000000000000000000000000000000..468dae2239619967a8940ed8e5ce07d0a2cd1aa5 --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs1:obs30:stride1:seed12/checkpoint_p0/checkpoint_000048840_25034752.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:03fce37730d8d2cfd5cbab65a10dda0fde99dbcdce761a25a91a8d07cbf4b483 +size 7081533 diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs1:obs30:stride1:seed12/config.json b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs1:obs30:stride1:seed12/config.json new file mode 100644 index 0000000000000000000000000000000000000000..2da696492f1546e15345d865c9e967b01c65e024 --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs1:obs30:stride1:seed12/config.json @@ -0,0 +1,266 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_fixed_l2_fs1_seed12", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 12, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs1-fixed_l2", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "fixed", + "fixed_l2", + "fs1", + "seed12" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "screen_size": "", + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "score_limit": -1, + "frame_stack": 1, + "debug": false, + "debug_timelimit_diagnostics": false, + "max_episode_steps": 0, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": null, + "max_latency_ms": null, + "latency_seed": null, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": false, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs1_seed12/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_flappy --experiment flappy_frame_stack_fixed_l2_fs1_seed12 --train_dir results/checkpoints_factor_sweeps/flappy/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-render-batch-size 128 --seed 12 --episode_metrics_path results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs1_seed12/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 2 --num_envs_per_worker 1 --num_policies 1 --batch_size 4096 --rollout 128 --recurrence 1 --num_epochs 8 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 400 --learning_rate 0.00025 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.5 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --use_rnn False --normalize_returns True --normalize_input True --latency-type fixed --fixed-latency-ms 66.66666666666667 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --with_wandb True --wandb_project latency-sensitive-bench --wandb_group flappy-fs1-fixed_l2 --wandb_job_type sample_factory --wandb_tags factor_sweep flappy frame_stack fixed fixed_l2 fs1 seed12 --gym_id FlappyBird-v0 --env-fps 30 --obs-fps 30.0 --use_lidar False --normalize_obs True --audio_on False --obs_resize 84,84 --use-gpu-render True --simulator gpu --gpu-render-device auto --gpu-render-profile False --gpu-render-profile-interval 200 --pipe_gap 100 --bird_color yellow --pipe_color green --background day --frame_stack 1 --debug False --debug-timelimit-diagnostics False --hold-policy one_frame_then_noop --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_fixed_l2_fs1_seed12", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 12, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs1-fixed_l2", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "fixed", + "fixed_l2", + "fs1", + "seed12" + ], + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "frame_stack": 1, + "debug": false, + "debug_timelimit_diagnostics": false, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "add_latency_info": false, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs1_seed12/episode_metrics.jsonl" + }, + "git_hash": "eb3a2e1efbd2aa03a60d7f44f5e18d8fdd0f5a2d", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/flappy/context_window/train/frame_stack/fixed_l2/fs1/seed_12", + "wandb_unique_id": "flappy-fs1-fixed_l2-s12" +} \ No newline at end of file diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs1:obs30:stride1:seed13/checkpoint_p0/checkpoint_000048840_25034752.pth b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs1:obs30:stride1:seed13/checkpoint_p0/checkpoint_000048840_25034752.pth new file mode 100644 index 0000000000000000000000000000000000000000..57e06b5f9a68594d3f788e806199a851b454c9e3 --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs1:obs30:stride1:seed13/checkpoint_p0/checkpoint_000048840_25034752.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ca1f3609d92160a88d0b64e873f16eeac4fe20e13352d8ecfa0e5068e8cbfb0a +size 7081533 diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs1:obs30:stride1:seed13/config.json b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs1:obs30:stride1:seed13/config.json new file mode 100644 index 0000000000000000000000000000000000000000..26b35f6e00a790295c04648503b41bcba8474006 --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs1:obs30:stride1:seed13/config.json @@ -0,0 +1,266 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_fixed_l2_fs1_seed13", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 13, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs1-fixed_l2", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "fixed", + "fixed_l2", + "fs1", + "seed13" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "screen_size": "", + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "score_limit": -1, + "frame_stack": 1, + "debug": false, + "debug_timelimit_diagnostics": false, + "max_episode_steps": 0, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": null, + "max_latency_ms": null, + "latency_seed": null, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": false, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs1_seed13/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_flappy --experiment flappy_frame_stack_fixed_l2_fs1_seed13 --train_dir results/checkpoints_factor_sweeps/flappy/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-render-batch-size 128 --seed 13 --episode_metrics_path results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs1_seed13/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 2 --num_envs_per_worker 1 --num_policies 1 --batch_size 4096 --rollout 128 --recurrence 1 --num_epochs 8 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 400 --learning_rate 0.00025 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.5 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --use_rnn False --normalize_returns True --normalize_input True --latency-type fixed --fixed-latency-ms 66.66666666666667 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --with_wandb True --wandb_project latency-sensitive-bench --wandb_group flappy-fs1-fixed_l2 --wandb_job_type sample_factory --wandb_tags factor_sweep flappy frame_stack fixed fixed_l2 fs1 seed13 --gym_id FlappyBird-v0 --env-fps 30 --obs-fps 30.0 --use_lidar False --normalize_obs True --audio_on False --obs_resize 84,84 --use-gpu-render True --simulator gpu --gpu-render-device auto --gpu-render-profile False --gpu-render-profile-interval 200 --pipe_gap 100 --bird_color yellow --pipe_color green --background day --frame_stack 1 --debug False --debug-timelimit-diagnostics False --hold-policy one_frame_then_noop --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_fixed_l2_fs1_seed13", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 13, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs1-fixed_l2", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "fixed", + "fixed_l2", + "fs1", + "seed13" + ], + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "frame_stack": 1, + "debug": false, + "debug_timelimit_diagnostics": false, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "add_latency_info": false, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs1_seed13/episode_metrics.jsonl" + }, + "git_hash": "eb3a2e1efbd2aa03a60d7f44f5e18d8fdd0f5a2d", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/flappy/context_window/train/frame_stack/fixed_l2/fs1/seed_13", + "wandb_unique_id": "flappy-fs1-fixed_l2-s13" +} \ No newline at end of file diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs1:obs30:stride1:seed14/checkpoint_p0/checkpoint_000048840_25034752.pth b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs1:obs30:stride1:seed14/checkpoint_p0/checkpoint_000048840_25034752.pth new file mode 100644 index 0000000000000000000000000000000000000000..1ee858bda6986844630c57b73f8ea313ead55dee --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs1:obs30:stride1:seed14/checkpoint_p0/checkpoint_000048840_25034752.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4f7ab6ec41ab166964d8a1cef450afdf19eb3878b890c3c0fef51eaaea478783 +size 7081533 diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs1:obs30:stride1:seed14/config.json b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs1:obs30:stride1:seed14/config.json new file mode 100644 index 0000000000000000000000000000000000000000..127f94ba2e3f83ce586cdeaf302ccd780eb0b683 --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs1:obs30:stride1:seed14/config.json @@ -0,0 +1,266 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_fixed_l2_fs1_seed14", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 14, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs1-fixed_l2", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "fixed", + "fixed_l2", + "fs1", + "seed14" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "screen_size": "", + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "score_limit": -1, + "frame_stack": 1, + "debug": false, + "debug_timelimit_diagnostics": false, + "max_episode_steps": 0, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": null, + "max_latency_ms": null, + "latency_seed": null, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": false, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs1_seed14/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_flappy --experiment flappy_frame_stack_fixed_l2_fs1_seed14 --train_dir results/checkpoints_factor_sweeps/flappy/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-render-batch-size 128 --seed 14 --episode_metrics_path results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs1_seed14/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 2 --num_envs_per_worker 1 --num_policies 1 --batch_size 4096 --rollout 128 --recurrence 1 --num_epochs 8 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 400 --learning_rate 0.00025 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.5 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --use_rnn False --normalize_returns True --normalize_input True --latency-type fixed --fixed-latency-ms 66.66666666666667 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --with_wandb True --wandb_project latency-sensitive-bench --wandb_group flappy-fs1-fixed_l2 --wandb_job_type sample_factory --wandb_tags factor_sweep flappy frame_stack fixed fixed_l2 fs1 seed14 --gym_id FlappyBird-v0 --env-fps 30 --obs-fps 30.0 --use_lidar False --normalize_obs True --audio_on False --obs_resize 84,84 --use-gpu-render True --simulator gpu --gpu-render-device auto --gpu-render-profile False --gpu-render-profile-interval 200 --pipe_gap 100 --bird_color yellow --pipe_color green --background day --frame_stack 1 --debug False --debug-timelimit-diagnostics False --hold-policy one_frame_then_noop --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_fixed_l2_fs1_seed14", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 14, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs1-fixed_l2", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "fixed", + "fixed_l2", + "fs1", + "seed14" + ], + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "frame_stack": 1, + "debug": false, + "debug_timelimit_diagnostics": false, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "add_latency_info": false, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs1_seed14/episode_metrics.jsonl" + }, + "git_hash": "eb3a2e1efbd2aa03a60d7f44f5e18d8fdd0f5a2d", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/flappy/context_window/train/frame_stack/fixed_l2/fs1/seed_14", + "wandb_unique_id": "flappy-fs1-fixed_l2-s14" +} \ No newline at end of file diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs2:obs30:stride1:seed10/checkpoint_p0/checkpoint_000048840_25034752.pth b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs2:obs30:stride1:seed10/checkpoint_p0/checkpoint_000048840_25034752.pth new file mode 100644 index 0000000000000000000000000000000000000000..f84458c4950195abd024309c9f3c25bf715684e3 --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs2:obs30:stride1:seed10/checkpoint_p0/checkpoint_000048840_25034752.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a4c12317ece8b96696d45fd3645865aa39c0d3b3bbef2e4e59b468bed8c02aef +size 7444797 diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs2:obs30:stride1:seed10/config.json b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs2:obs30:stride1:seed10/config.json new file mode 100644 index 0000000000000000000000000000000000000000..cee752a14eb83b4449acc6756410e3be1a2d36e2 --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs2:obs30:stride1:seed10/config.json @@ -0,0 +1,266 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_fixed_l2_fs2_seed10", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 10, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs2-fixed_l2", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "fixed", + "fixed_l2", + "fs2", + "seed10" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "screen_size": "", + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "score_limit": -1, + "frame_stack": 2, + "debug": false, + "debug_timelimit_diagnostics": false, + "max_episode_steps": 0, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": null, + "max_latency_ms": null, + "latency_seed": null, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": false, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs2_seed10/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_flappy --experiment flappy_frame_stack_fixed_l2_fs2_seed10 --train_dir results/checkpoints_factor_sweeps/flappy/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-render-batch-size 128 --seed 10 --episode_metrics_path results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs2_seed10/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 2 --num_envs_per_worker 1 --num_policies 1 --batch_size 4096 --rollout 128 --recurrence 1 --num_epochs 8 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 400 --learning_rate 0.00025 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.5 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --use_rnn False --normalize_returns True --normalize_input True --latency-type fixed --fixed-latency-ms 66.66666666666667 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --with_wandb True --wandb_project latency-sensitive-bench --wandb_group flappy-fs2-fixed_l2 --wandb_job_type sample_factory --wandb_tags factor_sweep flappy frame_stack fixed fixed_l2 fs2 seed10 --gym_id FlappyBird-v0 --env-fps 30 --obs-fps 30.0 --use_lidar False --normalize_obs True --audio_on False --obs_resize 84,84 --use-gpu-render True --simulator gpu --gpu-render-device auto --gpu-render-profile False --gpu-render-profile-interval 200 --pipe_gap 100 --bird_color yellow --pipe_color green --background day --frame_stack 2 --debug False --debug-timelimit-diagnostics False --hold-policy one_frame_then_noop --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_fixed_l2_fs2_seed10", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 10, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs2-fixed_l2", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "fixed", + "fixed_l2", + "fs2", + "seed10" + ], + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "frame_stack": 2, + "debug": false, + "debug_timelimit_diagnostics": false, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "add_latency_info": false, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs2_seed10/episode_metrics.jsonl" + }, + "git_hash": "eb3a2e1efbd2aa03a60d7f44f5e18d8fdd0f5a2d", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/flappy/context_window/train/frame_stack/fixed_l2/fs2/seed_10", + "wandb_unique_id": "flappy-fs2-fixed_l2-s10" +} \ No newline at end of file diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs2:obs30:stride1:seed11/checkpoint_p0/checkpoint_000048840_25034752.pth b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs2:obs30:stride1:seed11/checkpoint_p0/checkpoint_000048840_25034752.pth new file mode 100644 index 0000000000000000000000000000000000000000..63b3b59cbc246f924e48b67df5dba49bfb543f0f --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs2:obs30:stride1:seed11/checkpoint_p0/checkpoint_000048840_25034752.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ff346e95bd7a6fdc11a4036ba6c71bf9fcb4a66326619993f31a6fe264609be +size 7444797 diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs2:obs30:stride1:seed11/config.json b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs2:obs30:stride1:seed11/config.json new file mode 100644 index 0000000000000000000000000000000000000000..addd145f062ca0871b4cc80f8e7f4bc72f0e7cd1 --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs2:obs30:stride1:seed11/config.json @@ -0,0 +1,266 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_fixed_l2_fs2_seed11", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 11, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs2-fixed_l2", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "fixed", + "fixed_l2", + "fs2", + "seed11" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "screen_size": "", + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "score_limit": -1, + "frame_stack": 2, + "debug": false, + "debug_timelimit_diagnostics": false, + "max_episode_steps": 0, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": null, + "max_latency_ms": null, + "latency_seed": null, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": false, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs2_seed11/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_flappy --experiment flappy_frame_stack_fixed_l2_fs2_seed11 --train_dir results/checkpoints_factor_sweeps/flappy/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-render-batch-size 128 --seed 11 --episode_metrics_path results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs2_seed11/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 2 --num_envs_per_worker 1 --num_policies 1 --batch_size 4096 --rollout 128 --recurrence 1 --num_epochs 8 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 400 --learning_rate 0.00025 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.5 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --use_rnn False --normalize_returns True --normalize_input True --latency-type fixed --fixed-latency-ms 66.66666666666667 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --with_wandb True --wandb_project latency-sensitive-bench --wandb_group flappy-fs2-fixed_l2 --wandb_job_type sample_factory --wandb_tags factor_sweep flappy frame_stack fixed fixed_l2 fs2 seed11 --gym_id FlappyBird-v0 --env-fps 30 --obs-fps 30.0 --use_lidar False --normalize_obs True --audio_on False --obs_resize 84,84 --use-gpu-render True --simulator gpu --gpu-render-device auto --gpu-render-profile False --gpu-render-profile-interval 200 --pipe_gap 100 --bird_color yellow --pipe_color green --background day --frame_stack 2 --debug False --debug-timelimit-diagnostics False --hold-policy one_frame_then_noop --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_fixed_l2_fs2_seed11", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 11, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs2-fixed_l2", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "fixed", + "fixed_l2", + "fs2", + "seed11" + ], + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "frame_stack": 2, + "debug": false, + "debug_timelimit_diagnostics": false, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "add_latency_info": false, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs2_seed11/episode_metrics.jsonl" + }, + "git_hash": "eb3a2e1efbd2aa03a60d7f44f5e18d8fdd0f5a2d", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/flappy/context_window/train/frame_stack/fixed_l2/fs2/seed_11", + "wandb_unique_id": "flappy-fs2-fixed_l2-s11" +} \ No newline at end of file diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs2:obs30:stride1:seed12/checkpoint_p0/checkpoint_000048776_25034752.pth b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs2:obs30:stride1:seed12/checkpoint_p0/checkpoint_000048776_25034752.pth new file mode 100644 index 0000000000000000000000000000000000000000..24948015f7a74ea8180538a3ec2d3c5fa161b0a1 --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs2:obs30:stride1:seed12/checkpoint_p0/checkpoint_000048776_25034752.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:625665ca09b6d8c9241a99e9f2e91513514f008bafb7d235002a8166f53fa156 +size 7444797 diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs2:obs30:stride1:seed12/config.json b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs2:obs30:stride1:seed12/config.json new file mode 100644 index 0000000000000000000000000000000000000000..050796b6b4a5f042b4a5b13e2f2604d034837303 --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs2:obs30:stride1:seed12/config.json @@ -0,0 +1,266 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_fixed_l2_fs2_seed12", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 12, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs2-fixed_l2", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "fixed", + "fixed_l2", + "fs2", + "seed12" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "screen_size": "", + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "score_limit": -1, + "frame_stack": 2, + "debug": false, + "debug_timelimit_diagnostics": false, + "max_episode_steps": 0, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": null, + "max_latency_ms": null, + "latency_seed": null, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": false, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs2_seed12/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_flappy --experiment flappy_frame_stack_fixed_l2_fs2_seed12 --train_dir results/checkpoints_factor_sweeps/flappy/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-render-batch-size 128 --seed 12 --episode_metrics_path results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs2_seed12/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 2 --num_envs_per_worker 1 --num_policies 1 --batch_size 4096 --rollout 128 --recurrence 1 --num_epochs 8 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 400 --learning_rate 0.00025 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.5 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --use_rnn False --normalize_returns True --normalize_input True --latency-type fixed --fixed-latency-ms 66.66666666666667 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --with_wandb True --wandb_project latency-sensitive-bench --wandb_group flappy-fs2-fixed_l2 --wandb_job_type sample_factory --wandb_tags factor_sweep flappy frame_stack fixed fixed_l2 fs2 seed12 --gym_id FlappyBird-v0 --env-fps 30 --obs-fps 30.0 --use_lidar False --normalize_obs True --audio_on False --obs_resize 84,84 --use-gpu-render True --simulator gpu --gpu-render-device auto --gpu-render-profile False --gpu-render-profile-interval 200 --pipe_gap 100 --bird_color yellow --pipe_color green --background day --frame_stack 2 --debug False --debug-timelimit-diagnostics False --hold-policy one_frame_then_noop --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_fixed_l2_fs2_seed12", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 12, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs2-fixed_l2", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "fixed", + "fixed_l2", + "fs2", + "seed12" + ], + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "frame_stack": 2, + "debug": false, + "debug_timelimit_diagnostics": false, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "add_latency_info": false, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs2_seed12/episode_metrics.jsonl" + }, + "git_hash": "eb3a2e1efbd2aa03a60d7f44f5e18d8fdd0f5a2d", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/flappy/context_window/train/frame_stack/fixed_l2/fs2/seed_12", + "wandb_unique_id": "flappy-fs2-fixed_l2-s12" +} \ No newline at end of file diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs2:obs30:stride1:seed13/checkpoint_p0/checkpoint_000048808_25034752.pth b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs2:obs30:stride1:seed13/checkpoint_p0/checkpoint_000048808_25034752.pth new file mode 100644 index 0000000000000000000000000000000000000000..326de2ce7dbc6366a524504f1483c94db77c7e2f --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs2:obs30:stride1:seed13/checkpoint_p0/checkpoint_000048808_25034752.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ed2b6390d6f0150e461a6b0e4f7dd3bb1d4b1d3e9b416f304e1697867b29e3a3 +size 7444797 diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs2:obs30:stride1:seed13/config.json b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs2:obs30:stride1:seed13/config.json new file mode 100644 index 0000000000000000000000000000000000000000..56f4aa873d09d6bf03b1f0035c7dda620c95af6c --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs2:obs30:stride1:seed13/config.json @@ -0,0 +1,266 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_fixed_l2_fs2_seed13", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 13, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs2-fixed_l2", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "fixed", + "fixed_l2", + "fs2", + "seed13" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "screen_size": "", + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "score_limit": -1, + "frame_stack": 2, + "debug": false, + "debug_timelimit_diagnostics": false, + "max_episode_steps": 0, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": null, + "max_latency_ms": null, + "latency_seed": null, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": false, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs2_seed13/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_flappy --experiment flappy_frame_stack_fixed_l2_fs2_seed13 --train_dir results/checkpoints_factor_sweeps/flappy/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-render-batch-size 128 --seed 13 --episode_metrics_path results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs2_seed13/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 2 --num_envs_per_worker 1 --num_policies 1 --batch_size 4096 --rollout 128 --recurrence 1 --num_epochs 8 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 400 --learning_rate 0.00025 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.5 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --use_rnn False --normalize_returns True --normalize_input True --latency-type fixed --fixed-latency-ms 66.66666666666667 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --with_wandb True --wandb_project latency-sensitive-bench --wandb_group flappy-fs2-fixed_l2 --wandb_job_type sample_factory --wandb_tags factor_sweep flappy frame_stack fixed fixed_l2 fs2 seed13 --gym_id FlappyBird-v0 --env-fps 30 --obs-fps 30.0 --use_lidar False --normalize_obs True --audio_on False --obs_resize 84,84 --use-gpu-render True --simulator gpu --gpu-render-device auto --gpu-render-profile False --gpu-render-profile-interval 200 --pipe_gap 100 --bird_color yellow --pipe_color green --background day --frame_stack 2 --debug False --debug-timelimit-diagnostics False --hold-policy one_frame_then_noop --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_fixed_l2_fs2_seed13", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 13, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs2-fixed_l2", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "fixed", + "fixed_l2", + "fs2", + "seed13" + ], + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "frame_stack": 2, + "debug": false, + "debug_timelimit_diagnostics": false, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "add_latency_info": false, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs2_seed13/episode_metrics.jsonl" + }, + "git_hash": "eb3a2e1efbd2aa03a60d7f44f5e18d8fdd0f5a2d", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/flappy/context_window/train/frame_stack/fixed_l2/fs2/seed_13", + "wandb_unique_id": "flappy-fs2-fixed_l2-s13" +} \ No newline at end of file diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs2:obs30:stride1:seed14/checkpoint_p0/checkpoint_000048736_25034752.pth b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs2:obs30:stride1:seed14/checkpoint_p0/checkpoint_000048736_25034752.pth new file mode 100644 index 0000000000000000000000000000000000000000..c95a733e24ad239081cac076a4ff75fe4f172e2d --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs2:obs30:stride1:seed14/checkpoint_p0/checkpoint_000048736_25034752.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:612817274f8a934b8c3c5a452746d27cb7a47f046b5ed7ac6b544bf8f38b6cc1 +size 7444797 diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs2:obs30:stride1:seed14/config.json b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs2:obs30:stride1:seed14/config.json new file mode 100644 index 0000000000000000000000000000000000000000..1ee6cbfca0cfed9c6e50088b7c570268619315b6 --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs2:obs30:stride1:seed14/config.json @@ -0,0 +1,266 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_fixed_l2_fs2_seed14", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 14, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs2-fixed_l2", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "fixed", + "fixed_l2", + "fs2", + "seed14" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "screen_size": "", + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "score_limit": -1, + "frame_stack": 2, + "debug": false, + "debug_timelimit_diagnostics": false, + "max_episode_steps": 0, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": null, + "max_latency_ms": null, + "latency_seed": null, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": false, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs2_seed14/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_flappy --experiment flappy_frame_stack_fixed_l2_fs2_seed14 --train_dir results/checkpoints_factor_sweeps/flappy/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-render-batch-size 128 --seed 14 --episode_metrics_path results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs2_seed14/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 2 --num_envs_per_worker 1 --num_policies 1 --batch_size 4096 --rollout 128 --recurrence 1 --num_epochs 8 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 400 --learning_rate 0.00025 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.5 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --use_rnn False --normalize_returns True --normalize_input True --latency-type fixed --fixed-latency-ms 66.66666666666667 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --with_wandb True --wandb_project latency-sensitive-bench --wandb_group flappy-fs2-fixed_l2 --wandb_job_type sample_factory --wandb_tags factor_sweep flappy frame_stack fixed fixed_l2 fs2 seed14 --gym_id FlappyBird-v0 --env-fps 30 --obs-fps 30.0 --use_lidar False --normalize_obs True --audio_on False --obs_resize 84,84 --use-gpu-render True --simulator gpu --gpu-render-device auto --gpu-render-profile False --gpu-render-profile-interval 200 --pipe_gap 100 --bird_color yellow --pipe_color green --background day --frame_stack 2 --debug False --debug-timelimit-diagnostics False --hold-policy one_frame_then_noop --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_fixed_l2_fs2_seed14", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 14, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs2-fixed_l2", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "fixed", + "fixed_l2", + "fs2", + "seed14" + ], + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "frame_stack": 2, + "debug": false, + "debug_timelimit_diagnostics": false, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "add_latency_info": false, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs2_seed14/episode_metrics.jsonl" + }, + "git_hash": "eb3a2e1efbd2aa03a60d7f44f5e18d8fdd0f5a2d", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/flappy/context_window/train/frame_stack/fixed_l2/fs2/seed_14", + "wandb_unique_id": "flappy-fs2-fixed_l2-s14" +} \ No newline at end of file diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs3:obs30:stride1:seed10/checkpoint_p0/checkpoint_000048840_25034752.pth b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs3:obs30:stride1:seed10/checkpoint_p0/checkpoint_000048840_25034752.pth new file mode 100644 index 0000000000000000000000000000000000000000..5c7712671c9177f70ce5d4063234d48c804ee7b7 --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs3:obs30:stride1:seed10/checkpoint_p0/checkpoint_000048840_25034752.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bf85f4834ab3ba3c45f0ffc9a2a2241934ecc0e7f8bd2056b93502da0e28bddb +size 7808061 diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs3:obs30:stride1:seed10/config.json b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs3:obs30:stride1:seed10/config.json new file mode 100644 index 0000000000000000000000000000000000000000..0d93bc2a8b2c36725b7627c9cdaf6c289bc85e9a --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs3:obs30:stride1:seed10/config.json @@ -0,0 +1,266 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_fixed_l2_fs3_seed10", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 10, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs3-fixed_l2", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "fixed", + "fixed_l2", + "fs3", + "seed10" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "screen_size": "", + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "score_limit": -1, + "frame_stack": 3, + "debug": false, + "debug_timelimit_diagnostics": false, + "max_episode_steps": 0, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": null, + "max_latency_ms": null, + "latency_seed": null, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": false, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs3_seed10/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_flappy --experiment flappy_frame_stack_fixed_l2_fs3_seed10 --train_dir results/checkpoints_factor_sweeps/flappy/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-render-batch-size 128 --seed 10 --episode_metrics_path results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs3_seed10/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 2 --num_envs_per_worker 1 --num_policies 1 --batch_size 4096 --rollout 128 --recurrence 1 --num_epochs 8 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 400 --learning_rate 0.00025 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.5 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --use_rnn False --normalize_returns True --normalize_input True --latency-type fixed --fixed-latency-ms 66.66666666666667 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --with_wandb True --wandb_project latency-sensitive-bench --wandb_group flappy-fs3-fixed_l2 --wandb_job_type sample_factory --wandb_tags factor_sweep flappy frame_stack fixed fixed_l2 fs3 seed10 --gym_id FlappyBird-v0 --env-fps 30 --obs-fps 30.0 --use_lidar False --normalize_obs True --audio_on False --obs_resize 84,84 --use-gpu-render True --simulator gpu --gpu-render-device auto --gpu-render-profile False --gpu-render-profile-interval 200 --pipe_gap 100 --bird_color yellow --pipe_color green --background day --frame_stack 3 --debug False --debug-timelimit-diagnostics False --hold-policy one_frame_then_noop --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_fixed_l2_fs3_seed10", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 10, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs3-fixed_l2", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "fixed", + "fixed_l2", + "fs3", + "seed10" + ], + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "frame_stack": 3, + "debug": false, + "debug_timelimit_diagnostics": false, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "add_latency_info": false, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs3_seed10/episode_metrics.jsonl" + }, + "git_hash": "eb3a2e1efbd2aa03a60d7f44f5e18d8fdd0f5a2d", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/flappy/context_window/train/frame_stack/fixed_l2/fs3/seed_10", + "wandb_unique_id": "flappy-fs3-fixed_l2-s10" +} \ No newline at end of file diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs3:obs30:stride1:seed11/checkpoint_p0/checkpoint_000048832_25034752.pth b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs3:obs30:stride1:seed11/checkpoint_p0/checkpoint_000048832_25034752.pth new file mode 100644 index 0000000000000000000000000000000000000000..c5df731b1fe61ad31c8b276cd9b6f302c0e1e7a3 --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs3:obs30:stride1:seed11/checkpoint_p0/checkpoint_000048832_25034752.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:424e4faa9060bd5297619dc81ab653808eba30bc7a4a215208619f4c7e311f96 +size 7808061 diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs3:obs30:stride1:seed11/config.json b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs3:obs30:stride1:seed11/config.json new file mode 100644 index 0000000000000000000000000000000000000000..f2be0d3cf7ad6c51b8c146e5314c7e4f7cf15c74 --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs3:obs30:stride1:seed11/config.json @@ -0,0 +1,266 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_fixed_l2_fs3_seed11", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 11, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs3-fixed_l2", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "fixed", + "fixed_l2", + "fs3", + "seed11" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "screen_size": "", + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "score_limit": -1, + "frame_stack": 3, + "debug": false, + "debug_timelimit_diagnostics": false, + "max_episode_steps": 0, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": null, + "max_latency_ms": null, + "latency_seed": null, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": false, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs3_seed11/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_flappy --experiment flappy_frame_stack_fixed_l2_fs3_seed11 --train_dir results/checkpoints_factor_sweeps/flappy/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-render-batch-size 128 --seed 11 --episode_metrics_path results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs3_seed11/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 2 --num_envs_per_worker 1 --num_policies 1 --batch_size 4096 --rollout 128 --recurrence 1 --num_epochs 8 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 400 --learning_rate 0.00025 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.5 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --use_rnn False --normalize_returns True --normalize_input True --latency-type fixed --fixed-latency-ms 66.66666666666667 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --with_wandb True --wandb_project latency-sensitive-bench --wandb_group flappy-fs3-fixed_l2 --wandb_job_type sample_factory --wandb_tags factor_sweep flappy frame_stack fixed fixed_l2 fs3 seed11 --gym_id FlappyBird-v0 --env-fps 30 --obs-fps 30.0 --use_lidar False --normalize_obs True --audio_on False --obs_resize 84,84 --use-gpu-render True --simulator gpu --gpu-render-device auto --gpu-render-profile False --gpu-render-profile-interval 200 --pipe_gap 100 --bird_color yellow --pipe_color green --background day --frame_stack 3 --debug False --debug-timelimit-diagnostics False --hold-policy one_frame_then_noop --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_fixed_l2_fs3_seed11", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 11, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs3-fixed_l2", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "fixed", + "fixed_l2", + "fs3", + "seed11" + ], + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "frame_stack": 3, + "debug": false, + "debug_timelimit_diagnostics": false, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "add_latency_info": false, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs3_seed11/episode_metrics.jsonl" + }, + "git_hash": "eb3a2e1efbd2aa03a60d7f44f5e18d8fdd0f5a2d", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/flappy/context_window/train/frame_stack/fixed_l2/fs3/seed_11", + "wandb_unique_id": "flappy-fs3-fixed_l2-s11" +} \ No newline at end of file diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs3:obs30:stride1:seed12/checkpoint_p0/checkpoint_000048848_25034752.pth b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs3:obs30:stride1:seed12/checkpoint_p0/checkpoint_000048848_25034752.pth new file mode 100644 index 0000000000000000000000000000000000000000..9611e011edeaa283f758db909409b3e386c9fb83 --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs3:obs30:stride1:seed12/checkpoint_p0/checkpoint_000048848_25034752.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5697efae8003b7e07ec3a54b789e405242a6bf005599ef2adb905c75f086b484 +size 7808061 diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs3:obs30:stride1:seed12/config.json b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs3:obs30:stride1:seed12/config.json new file mode 100644 index 0000000000000000000000000000000000000000..d531ae531b6a05372ba98e1944b6f4639ff2458f --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs3:obs30:stride1:seed12/config.json @@ -0,0 +1,266 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_fixed_l2_fs3_seed12", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 12, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs3-fixed_l2", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "fixed", + "fixed_l2", + "fs3", + "seed12" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "screen_size": "", + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "score_limit": -1, + "frame_stack": 3, + "debug": false, + "debug_timelimit_diagnostics": false, + "max_episode_steps": 0, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": null, + "max_latency_ms": null, + "latency_seed": null, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": false, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs3_seed12/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_flappy --experiment flappy_frame_stack_fixed_l2_fs3_seed12 --train_dir results/checkpoints_factor_sweeps/flappy/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-render-batch-size 128 --seed 12 --episode_metrics_path results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs3_seed12/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 2 --num_envs_per_worker 1 --num_policies 1 --batch_size 4096 --rollout 128 --recurrence 1 --num_epochs 8 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 400 --learning_rate 0.00025 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.5 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --use_rnn False --normalize_returns True --normalize_input True --latency-type fixed --fixed-latency-ms 66.66666666666667 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --with_wandb True --wandb_project latency-sensitive-bench --wandb_group flappy-fs3-fixed_l2 --wandb_job_type sample_factory --wandb_tags factor_sweep flappy frame_stack fixed fixed_l2 fs3 seed12 --gym_id FlappyBird-v0 --env-fps 30 --obs-fps 30.0 --use_lidar False --normalize_obs True --audio_on False --obs_resize 84,84 --use-gpu-render True --simulator gpu --gpu-render-device auto --gpu-render-profile False --gpu-render-profile-interval 200 --pipe_gap 100 --bird_color yellow --pipe_color green --background day --frame_stack 3 --debug False --debug-timelimit-diagnostics False --hold-policy one_frame_then_noop --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_fixed_l2_fs3_seed12", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 12, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs3-fixed_l2", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "fixed", + "fixed_l2", + "fs3", + "seed12" + ], + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "frame_stack": 3, + "debug": false, + "debug_timelimit_diagnostics": false, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "add_latency_info": false, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs3_seed12/episode_metrics.jsonl" + }, + "git_hash": "eb3a2e1efbd2aa03a60d7f44f5e18d8fdd0f5a2d", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/flappy/context_window/train/frame_stack/fixed_l2/fs3/seed_12", + "wandb_unique_id": "flappy-fs3-fixed_l2-s12" +} \ No newline at end of file diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs3:obs30:stride1:seed13/checkpoint_p0/checkpoint_000048840_25034752.pth b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs3:obs30:stride1:seed13/checkpoint_p0/checkpoint_000048840_25034752.pth new file mode 100644 index 0000000000000000000000000000000000000000..0c7c385470e2db89ed842a3f88720ad9ba08c7ba --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs3:obs30:stride1:seed13/checkpoint_p0/checkpoint_000048840_25034752.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fe1796cb76c6acb8120e1506881b13ea862a9bd981215ba0bcee34c95163336a +size 7808061 diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs3:obs30:stride1:seed13/config.json b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs3:obs30:stride1:seed13/config.json new file mode 100644 index 0000000000000000000000000000000000000000..3cc81233fb35301ee6d745c06d9818398f868d9d --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs3:obs30:stride1:seed13/config.json @@ -0,0 +1,266 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_fixed_l2_fs3_seed13", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 13, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs3-fixed_l2", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "fixed", + "fixed_l2", + "fs3", + "seed13" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "screen_size": "", + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "score_limit": -1, + "frame_stack": 3, + "debug": false, + "debug_timelimit_diagnostics": false, + "max_episode_steps": 0, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": null, + "max_latency_ms": null, + "latency_seed": null, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": false, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs3_seed13/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_flappy --experiment flappy_frame_stack_fixed_l2_fs3_seed13 --train_dir results/checkpoints_factor_sweeps/flappy/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-render-batch-size 128 --seed 13 --episode_metrics_path results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs3_seed13/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 2 --num_envs_per_worker 1 --num_policies 1 --batch_size 4096 --rollout 128 --recurrence 1 --num_epochs 8 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 400 --learning_rate 0.00025 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.5 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --use_rnn False --normalize_returns True --normalize_input True --latency-type fixed --fixed-latency-ms 66.66666666666667 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --with_wandb True --wandb_project latency-sensitive-bench --wandb_group flappy-fs3-fixed_l2 --wandb_job_type sample_factory --wandb_tags factor_sweep flappy frame_stack fixed fixed_l2 fs3 seed13 --gym_id FlappyBird-v0 --env-fps 30 --obs-fps 30.0 --use_lidar False --normalize_obs True --audio_on False --obs_resize 84,84 --use-gpu-render True --simulator gpu --gpu-render-device auto --gpu-render-profile False --gpu-render-profile-interval 200 --pipe_gap 100 --bird_color yellow --pipe_color green --background day --frame_stack 3 --debug False --debug-timelimit-diagnostics False --hold-policy one_frame_then_noop --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_fixed_l2_fs3_seed13", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 13, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs3-fixed_l2", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "fixed", + "fixed_l2", + "fs3", + "seed13" + ], + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "frame_stack": 3, + "debug": false, + "debug_timelimit_diagnostics": false, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "add_latency_info": false, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs3_seed13/episode_metrics.jsonl" + }, + "git_hash": "eb3a2e1efbd2aa03a60d7f44f5e18d8fdd0f5a2d", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/flappy/context_window/train/frame_stack/fixed_l2/fs3/seed_13", + "wandb_unique_id": "flappy-fs3-fixed_l2-s13" +} \ No newline at end of file diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs3:obs30:stride1:seed14/checkpoint_p0/checkpoint_000048832_25034752.pth b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs3:obs30:stride1:seed14/checkpoint_p0/checkpoint_000048832_25034752.pth new file mode 100644 index 0000000000000000000000000000000000000000..2e95fbf0a3ad645691d557e74e2179862195ed1c --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs3:obs30:stride1:seed14/checkpoint_p0/checkpoint_000048832_25034752.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8873e430f42f86bc349c62d6a77ac0796028812029ede1daf60f606d7bb4327f +size 7808061 diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs3:obs30:stride1:seed14/config.json b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs3:obs30:stride1:seed14/config.json new file mode 100644 index 0000000000000000000000000000000000000000..fd45f57ba5281a1cd425b927436a60a96c38353b --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs3:obs30:stride1:seed14/config.json @@ -0,0 +1,266 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_fixed_l2_fs3_seed14", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 14, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs3-fixed_l2", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "fixed", + "fixed_l2", + "fs3", + "seed14" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "screen_size": "", + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "score_limit": -1, + "frame_stack": 3, + "debug": false, + "debug_timelimit_diagnostics": false, + "max_episode_steps": 0, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": null, + "max_latency_ms": null, + "latency_seed": null, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": false, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs3_seed14/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_flappy --experiment flappy_frame_stack_fixed_l2_fs3_seed14 --train_dir results/checkpoints_factor_sweeps/flappy/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-render-batch-size 128 --seed 14 --episode_metrics_path results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs3_seed14/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 2 --num_envs_per_worker 1 --num_policies 1 --batch_size 4096 --rollout 128 --recurrence 1 --num_epochs 8 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 400 --learning_rate 0.00025 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.5 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --use_rnn False --normalize_returns True --normalize_input True --latency-type fixed --fixed-latency-ms 66.66666666666667 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --with_wandb True --wandb_project latency-sensitive-bench --wandb_group flappy-fs3-fixed_l2 --wandb_job_type sample_factory --wandb_tags factor_sweep flappy frame_stack fixed fixed_l2 fs3 seed14 --gym_id FlappyBird-v0 --env-fps 30 --obs-fps 30.0 --use_lidar False --normalize_obs True --audio_on False --obs_resize 84,84 --use-gpu-render True --simulator gpu --gpu-render-device auto --gpu-render-profile False --gpu-render-profile-interval 200 --pipe_gap 100 --bird_color yellow --pipe_color green --background day --frame_stack 3 --debug False --debug-timelimit-diagnostics False --hold-policy one_frame_then_noop --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_fixed_l2_fs3_seed14", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 14, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs3-fixed_l2", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "fixed", + "fixed_l2", + "fs3", + "seed14" + ], + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "frame_stack": 3, + "debug": false, + "debug_timelimit_diagnostics": false, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "add_latency_info": false, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs3_seed14/episode_metrics.jsonl" + }, + "git_hash": "eb3a2e1efbd2aa03a60d7f44f5e18d8fdd0f5a2d", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/flappy/context_window/train/frame_stack/fixed_l2/fs3/seed_14", + "wandb_unique_id": "flappy-fs3-fixed_l2-s14" +} \ No newline at end of file diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs4:obs30:stride1:seed10/checkpoint_p0/checkpoint_000048840_25034752.pth b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs4:obs30:stride1:seed10/checkpoint_p0/checkpoint_000048840_25034752.pth new file mode 100644 index 0000000000000000000000000000000000000000..79e759d9958459137948b77bdac19384868168e5 --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs4:obs30:stride1:seed10/checkpoint_p0/checkpoint_000048840_25034752.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8f3f423a086e417941dcca9582e9f2a1de781bc6111cf50d082ba13785a72534 +size 8171325 diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs4:obs30:stride1:seed10/config.json b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs4:obs30:stride1:seed10/config.json new file mode 100644 index 0000000000000000000000000000000000000000..719b787093669bc832cec97017a9f4d0581092fe --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs4:obs30:stride1:seed10/config.json @@ -0,0 +1,266 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_fixed_l2_fs4_seed10", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 10, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs4-fixed_l2", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "fixed", + "fixed_l2", + "fs4", + "seed10" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "screen_size": "", + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "score_limit": -1, + "frame_stack": 4, + "debug": false, + "debug_timelimit_diagnostics": false, + "max_episode_steps": 0, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": null, + "max_latency_ms": null, + "latency_seed": null, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": false, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs4_seed10/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_flappy --experiment flappy_frame_stack_fixed_l2_fs4_seed10 --train_dir results/checkpoints_factor_sweeps/flappy/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-render-batch-size 128 --seed 10 --episode_metrics_path results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs4_seed10/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 2 --num_envs_per_worker 1 --num_policies 1 --batch_size 4096 --rollout 128 --recurrence 1 --num_epochs 8 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 400 --learning_rate 0.00025 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.5 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --use_rnn False --normalize_returns True --normalize_input True --latency-type fixed --fixed-latency-ms 66.66666666666667 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --with_wandb True --wandb_project latency-sensitive-bench --wandb_group flappy-fs4-fixed_l2 --wandb_job_type sample_factory --wandb_tags factor_sweep flappy frame_stack fixed fixed_l2 fs4 seed10 --gym_id FlappyBird-v0 --env-fps 30 --obs-fps 30.0 --use_lidar False --normalize_obs True --audio_on False --obs_resize 84,84 --use-gpu-render True --simulator gpu --gpu-render-device auto --gpu-render-profile False --gpu-render-profile-interval 200 --pipe_gap 100 --bird_color yellow --pipe_color green --background day --frame_stack 4 --debug False --debug-timelimit-diagnostics False --hold-policy one_frame_then_noop --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_fixed_l2_fs4_seed10", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 10, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs4-fixed_l2", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "fixed", + "fixed_l2", + "fs4", + "seed10" + ], + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "frame_stack": 4, + "debug": false, + "debug_timelimit_diagnostics": false, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "add_latency_info": false, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs4_seed10/episode_metrics.jsonl" + }, + "git_hash": "284fe8ace24f0e8a40c03c5b559969abd7caeb29", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/flappy/context_window/train/frame_stack/fixed_l2/fs4/seed_10", + "wandb_unique_id": "flappy-fs4-fixed_l2-s10" +} \ No newline at end of file diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs4:obs30:stride1:seed11/checkpoint_p0/checkpoint_000048848_25034752.pth b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs4:obs30:stride1:seed11/checkpoint_p0/checkpoint_000048848_25034752.pth new file mode 100644 index 0000000000000000000000000000000000000000..aa1f1227c324777d4fac40829b662ed11d42d906 --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs4:obs30:stride1:seed11/checkpoint_p0/checkpoint_000048848_25034752.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c9c1034ddb5fa480dcff4b8411f71b75045435b345a009816b7c152e7eeff0b6 +size 8171325 diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs4:obs30:stride1:seed11/config.json b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs4:obs30:stride1:seed11/config.json new file mode 100644 index 0000000000000000000000000000000000000000..d5ca4bc890517729dd2f9b6bc18935dba5723a9f --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs4:obs30:stride1:seed11/config.json @@ -0,0 +1,266 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_fixed_l2_fs4_seed11", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 11, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs4-fixed_l2", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "fixed", + "fixed_l2", + "fs4", + "seed11" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "screen_size": "", + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "score_limit": -1, + "frame_stack": 4, + "debug": false, + "debug_timelimit_diagnostics": false, + "max_episode_steps": 0, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": null, + "max_latency_ms": null, + "latency_seed": null, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": false, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs4_seed11/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_flappy --experiment flappy_frame_stack_fixed_l2_fs4_seed11 --train_dir results/checkpoints_factor_sweeps/flappy/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-render-batch-size 128 --seed 11 --episode_metrics_path results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs4_seed11/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 2 --num_envs_per_worker 1 --num_policies 1 --batch_size 4096 --rollout 128 --recurrence 1 --num_epochs 8 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 400 --learning_rate 0.00025 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.5 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --use_rnn False --normalize_returns True --normalize_input True --latency-type fixed --fixed-latency-ms 66.66666666666667 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --with_wandb True --wandb_project latency-sensitive-bench --wandb_group flappy-fs4-fixed_l2 --wandb_job_type sample_factory --wandb_tags factor_sweep flappy frame_stack fixed fixed_l2 fs4 seed11 --gym_id FlappyBird-v0 --env-fps 30 --obs-fps 30.0 --use_lidar False --normalize_obs True --audio_on False --obs_resize 84,84 --use-gpu-render True --simulator gpu --gpu-render-device auto --gpu-render-profile False --gpu-render-profile-interval 200 --pipe_gap 100 --bird_color yellow --pipe_color green --background day --frame_stack 4 --debug False --debug-timelimit-diagnostics False --hold-policy one_frame_then_noop --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_fixed_l2_fs4_seed11", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 11, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs4-fixed_l2", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "fixed", + "fixed_l2", + "fs4", + "seed11" + ], + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "frame_stack": 4, + "debug": false, + "debug_timelimit_diagnostics": false, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "add_latency_info": false, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs4_seed11/episode_metrics.jsonl" + }, + "git_hash": "284fe8ace24f0e8a40c03c5b559969abd7caeb29", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/flappy/context_window/train/frame_stack/fixed_l2/fs4/seed_11", + "wandb_unique_id": "flappy-fs4-fixed_l2-s11" +} \ No newline at end of file diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs4:obs30:stride1:seed12/checkpoint_p0/checkpoint_000048832_25034752.pth b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs4:obs30:stride1:seed12/checkpoint_p0/checkpoint_000048832_25034752.pth new file mode 100644 index 0000000000000000000000000000000000000000..b9a671c5de75cbc0229e8f9fee0b3c641fbff699 --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs4:obs30:stride1:seed12/checkpoint_p0/checkpoint_000048832_25034752.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:32ccd70a489d4a547e5211cbc17def8387fb34ff2226886febcd80780c9b7eb9 +size 8171325 diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs4:obs30:stride1:seed12/config.json b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs4:obs30:stride1:seed12/config.json new file mode 100644 index 0000000000000000000000000000000000000000..7cf3a3921612d7aed31d740788c548497b355eb3 --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs4:obs30:stride1:seed12/config.json @@ -0,0 +1,266 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_fixed_l2_fs4_seed12", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 12, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs4-fixed_l2", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "fixed", + "fixed_l2", + "fs4", + "seed12" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "screen_size": "", + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "score_limit": -1, + "frame_stack": 4, + "debug": false, + "debug_timelimit_diagnostics": false, + "max_episode_steps": 0, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": null, + "max_latency_ms": null, + "latency_seed": null, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": false, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs4_seed12/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_flappy --experiment flappy_frame_stack_fixed_l2_fs4_seed12 --train_dir results/checkpoints_factor_sweeps/flappy/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-render-batch-size 128 --seed 12 --episode_metrics_path results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs4_seed12/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 2 --num_envs_per_worker 1 --num_policies 1 --batch_size 4096 --rollout 128 --recurrence 1 --num_epochs 8 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 400 --learning_rate 0.00025 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.5 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --use_rnn False --normalize_returns True --normalize_input True --latency-type fixed --fixed-latency-ms 66.66666666666667 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --with_wandb True --wandb_project latency-sensitive-bench --wandb_group flappy-fs4-fixed_l2 --wandb_job_type sample_factory --wandb_tags factor_sweep flappy frame_stack fixed fixed_l2 fs4 seed12 --gym_id FlappyBird-v0 --env-fps 30 --obs-fps 30.0 --use_lidar False --normalize_obs True --audio_on False --obs_resize 84,84 --use-gpu-render True --simulator gpu --gpu-render-device auto --gpu-render-profile False --gpu-render-profile-interval 200 --pipe_gap 100 --bird_color yellow --pipe_color green --background day --frame_stack 4 --debug False --debug-timelimit-diagnostics False --hold-policy one_frame_then_noop --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_fixed_l2_fs4_seed12", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 12, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs4-fixed_l2", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "fixed", + "fixed_l2", + "fs4", + "seed12" + ], + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "frame_stack": 4, + "debug": false, + "debug_timelimit_diagnostics": false, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "add_latency_info": false, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs4_seed12/episode_metrics.jsonl" + }, + "git_hash": "284fe8ace24f0e8a40c03c5b559969abd7caeb29", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/flappy/context_window/train/frame_stack/fixed_l2/fs4/seed_12", + "wandb_unique_id": "flappy-fs4-fixed_l2-s12" +} \ No newline at end of file diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs4:obs30:stride1:seed13/checkpoint_p0/checkpoint_000048848_25034752.pth b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs4:obs30:stride1:seed13/checkpoint_p0/checkpoint_000048848_25034752.pth new file mode 100644 index 0000000000000000000000000000000000000000..65dc98a9cd1961ba9ee53e5e13b828d2a918fdf5 --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs4:obs30:stride1:seed13/checkpoint_p0/checkpoint_000048848_25034752.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:17370e985d20fd0146f72a4d6c7b4aa13ce60cea61c1cea5900734dc6a7965ae +size 8171325 diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs4:obs30:stride1:seed13/config.json b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs4:obs30:stride1:seed13/config.json new file mode 100644 index 0000000000000000000000000000000000000000..1f1257f74e2ea6695b8fde941dab4e52f0d68a10 --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs4:obs30:stride1:seed13/config.json @@ -0,0 +1,266 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_fixed_l2_fs4_seed13", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 13, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs4-fixed_l2", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "fixed", + "fixed_l2", + "fs4", + "seed13" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "screen_size": "", + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "score_limit": -1, + "frame_stack": 4, + "debug": false, + "debug_timelimit_diagnostics": false, + "max_episode_steps": 0, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": null, + "max_latency_ms": null, + "latency_seed": null, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": false, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs4_seed13/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_flappy --experiment flappy_frame_stack_fixed_l2_fs4_seed13 --train_dir results/checkpoints_factor_sweeps/flappy/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-render-batch-size 128 --seed 13 --episode_metrics_path results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs4_seed13/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 2 --num_envs_per_worker 1 --num_policies 1 --batch_size 4096 --rollout 128 --recurrence 1 --num_epochs 8 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 400 --learning_rate 0.00025 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.5 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --use_rnn False --normalize_returns True --normalize_input True --latency-type fixed --fixed-latency-ms 66.66666666666667 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --with_wandb True --wandb_project latency-sensitive-bench --wandb_group flappy-fs4-fixed_l2 --wandb_job_type sample_factory --wandb_tags factor_sweep flappy frame_stack fixed fixed_l2 fs4 seed13 --gym_id FlappyBird-v0 --env-fps 30 --obs-fps 30.0 --use_lidar False --normalize_obs True --audio_on False --obs_resize 84,84 --use-gpu-render True --simulator gpu --gpu-render-device auto --gpu-render-profile False --gpu-render-profile-interval 200 --pipe_gap 100 --bird_color yellow --pipe_color green --background day --frame_stack 4 --debug False --debug-timelimit-diagnostics False --hold-policy one_frame_then_noop --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_fixed_l2_fs4_seed13", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 13, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs4-fixed_l2", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "fixed", + "fixed_l2", + "fs4", + "seed13" + ], + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "frame_stack": 4, + "debug": false, + "debug_timelimit_diagnostics": false, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "add_latency_info": false, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs4_seed13/episode_metrics.jsonl" + }, + "git_hash": "284fe8ace24f0e8a40c03c5b559969abd7caeb29", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/flappy/context_window/train/frame_stack/fixed_l2/fs4/seed_13", + "wandb_unique_id": "flappy-fs4-fixed_l2-s13" +} \ No newline at end of file diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs4:obs30:stride1:seed14/checkpoint_p0/checkpoint_000048848_25034752.pth b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs4:obs30:stride1:seed14/checkpoint_p0/checkpoint_000048848_25034752.pth new file mode 100644 index 0000000000000000000000000000000000000000..da541801733bbd2b1782e1175bb1ed46ae0fd790 --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs4:obs30:stride1:seed14/checkpoint_p0/checkpoint_000048848_25034752.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e8b0fef1edba9abe37892f34f9a3d6c7ff23d64bb6282c98e2a5ef9679989393 +size 8171325 diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs4:obs30:stride1:seed14/config.json b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs4:obs30:stride1:seed14/config.json new file mode 100644 index 0000000000000000000000000000000000000000..3432d6fdca185dee06dee92c84f5f8d3555c4c57 --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs4:obs30:stride1:seed14/config.json @@ -0,0 +1,266 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_fixed_l2_fs4_seed14", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 14, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs4-fixed_l2", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "fixed", + "fixed_l2", + "fs4", + "seed14" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "screen_size": "", + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "score_limit": -1, + "frame_stack": 4, + "debug": false, + "debug_timelimit_diagnostics": false, + "max_episode_steps": 0, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": null, + "max_latency_ms": null, + "latency_seed": null, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": false, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs4_seed14/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_flappy --experiment flappy_frame_stack_fixed_l2_fs4_seed14 --train_dir results/checkpoints_factor_sweeps/flappy/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-render-batch-size 128 --seed 14 --episode_metrics_path results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs4_seed14/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 2 --num_envs_per_worker 1 --num_policies 1 --batch_size 4096 --rollout 128 --recurrence 1 --num_epochs 8 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 400 --learning_rate 0.00025 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.5 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --use_rnn False --normalize_returns True --normalize_input True --latency-type fixed --fixed-latency-ms 66.66666666666667 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --with_wandb True --wandb_project latency-sensitive-bench --wandb_group flappy-fs4-fixed_l2 --wandb_job_type sample_factory --wandb_tags factor_sweep flappy frame_stack fixed fixed_l2 fs4 seed14 --gym_id FlappyBird-v0 --env-fps 30 --obs-fps 30.0 --use_lidar False --normalize_obs True --audio_on False --obs_resize 84,84 --use-gpu-render True --simulator gpu --gpu-render-device auto --gpu-render-profile False --gpu-render-profile-interval 200 --pipe_gap 100 --bird_color yellow --pipe_color green --background day --frame_stack 4 --debug False --debug-timelimit-diagnostics False --hold-policy one_frame_then_noop --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_fixed_l2_fs4_seed14", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 14, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs4-fixed_l2", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "fixed", + "fixed_l2", + "fs4", + "seed14" + ], + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "frame_stack": 4, + "debug": false, + "debug_timelimit_diagnostics": false, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "add_latency_info": false, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs4_seed14/episode_metrics.jsonl" + }, + "git_hash": "284fe8ace24f0e8a40c03c5b559969abd7caeb29", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/flappy/context_window/train/frame_stack/fixed_l2/fs4/seed_14", + "wandb_unique_id": "flappy-fs4-fixed_l2-s14" +} \ No newline at end of file diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs5:obs30:stride1:seed10/checkpoint_p0/checkpoint_000048776_25034752.pth b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs5:obs30:stride1:seed10/checkpoint_p0/checkpoint_000048776_25034752.pth new file mode 100644 index 0000000000000000000000000000000000000000..964c9213e6217a943d47a63863eac3ba8bb14dea --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs5:obs30:stride1:seed10/checkpoint_p0/checkpoint_000048776_25034752.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0a458ea20badf2f8c8ed8ef69e7504b36cf475557608885aaf535e9a5fa9d585 +size 8534589 diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs5:obs30:stride1:seed10/config.json b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs5:obs30:stride1:seed10/config.json new file mode 100644 index 0000000000000000000000000000000000000000..152daf111c2956e4466a73ab45f83e475a10b3e0 --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs5:obs30:stride1:seed10/config.json @@ -0,0 +1,266 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_fixed_l2_fs5_seed10", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 10, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs5-fixed_l2", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "fixed", + "fixed_l2", + "fs5", + "seed10" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "screen_size": "", + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "score_limit": -1, + "frame_stack": 5, + "debug": false, + "debug_timelimit_diagnostics": false, + "max_episode_steps": 0, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": null, + "max_latency_ms": null, + "latency_seed": null, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": false, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs5_seed10/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_flappy --experiment flappy_frame_stack_fixed_l2_fs5_seed10 --train_dir results/checkpoints_factor_sweeps/flappy/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-render-batch-size 128 --seed 10 --episode_metrics_path results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs5_seed10/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 2 --num_envs_per_worker 1 --num_policies 1 --batch_size 4096 --rollout 128 --recurrence 1 --num_epochs 8 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 400 --learning_rate 0.00025 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.5 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --use_rnn False --normalize_returns True --normalize_input True --latency-type fixed --fixed-latency-ms 66.66666666666667 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --with_wandb True --wandb_project latency-sensitive-bench --wandb_group flappy-fs5-fixed_l2 --wandb_job_type sample_factory --wandb_tags factor_sweep flappy frame_stack fixed fixed_l2 fs5 seed10 --gym_id FlappyBird-v0 --env-fps 30 --obs-fps 30.0 --use_lidar False --normalize_obs True --audio_on False --obs_resize 84,84 --use-gpu-render True --simulator gpu --gpu-render-device auto --gpu-render-profile False --gpu-render-profile-interval 200 --pipe_gap 100 --bird_color yellow --pipe_color green --background day --frame_stack 5 --debug False --debug-timelimit-diagnostics False --hold-policy one_frame_then_noop --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_fixed_l2_fs5_seed10", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 10, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs5-fixed_l2", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "fixed", + "fixed_l2", + "fs5", + "seed10" + ], + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "frame_stack": 5, + "debug": false, + "debug_timelimit_diagnostics": false, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "add_latency_info": false, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs5_seed10/episode_metrics.jsonl" + }, + "git_hash": "284fe8ace24f0e8a40c03c5b559969abd7caeb29", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/flappy/context_window/train/frame_stack/fixed_l2/fs5/seed_10", + "wandb_unique_id": "flappy-fs5-fixed_l2-s10" +} \ No newline at end of file diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs5:obs30:stride1:seed11/checkpoint_p0/checkpoint_000048840_25034752.pth b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs5:obs30:stride1:seed11/checkpoint_p0/checkpoint_000048840_25034752.pth new file mode 100644 index 0000000000000000000000000000000000000000..b130ec8297d1282a697d2af8c9ca1b6fb02cc1aa --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs5:obs30:stride1:seed11/checkpoint_p0/checkpoint_000048840_25034752.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c4860378619c815d573a6ebe637a6ad50c45bdc2465e3d0cc41175b1d067246d +size 8534589 diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs5:obs30:stride1:seed11/config.json b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs5:obs30:stride1:seed11/config.json new file mode 100644 index 0000000000000000000000000000000000000000..b0610eed85189d2c8845d7a255552f3f1cdeac23 --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs5:obs30:stride1:seed11/config.json @@ -0,0 +1,266 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_fixed_l2_fs5_seed11", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 11, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs5-fixed_l2", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "fixed", + "fixed_l2", + "fs5", + "seed11" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "screen_size": "", + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "score_limit": -1, + "frame_stack": 5, + "debug": false, + "debug_timelimit_diagnostics": false, + "max_episode_steps": 0, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": null, + "max_latency_ms": null, + "latency_seed": null, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": false, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs5_seed11/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_flappy --experiment flappy_frame_stack_fixed_l2_fs5_seed11 --train_dir results/checkpoints_factor_sweeps/flappy/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-render-batch-size 128 --seed 11 --episode_metrics_path results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs5_seed11/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 2 --num_envs_per_worker 1 --num_policies 1 --batch_size 4096 --rollout 128 --recurrence 1 --num_epochs 8 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 400 --learning_rate 0.00025 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.5 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --use_rnn False --normalize_returns True --normalize_input True --latency-type fixed --fixed-latency-ms 66.66666666666667 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --with_wandb True --wandb_project latency-sensitive-bench --wandb_group flappy-fs5-fixed_l2 --wandb_job_type sample_factory --wandb_tags factor_sweep flappy frame_stack fixed fixed_l2 fs5 seed11 --gym_id FlappyBird-v0 --env-fps 30 --obs-fps 30.0 --use_lidar False --normalize_obs True --audio_on False --obs_resize 84,84 --use-gpu-render True --simulator gpu --gpu-render-device auto --gpu-render-profile False --gpu-render-profile-interval 200 --pipe_gap 100 --bird_color yellow --pipe_color green --background day --frame_stack 5 --debug False --debug-timelimit-diagnostics False --hold-policy one_frame_then_noop --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_fixed_l2_fs5_seed11", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 11, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs5-fixed_l2", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "fixed", + "fixed_l2", + "fs5", + "seed11" + ], + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "frame_stack": 5, + "debug": false, + "debug_timelimit_diagnostics": false, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "add_latency_info": false, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs5_seed11/episode_metrics.jsonl" + }, + "git_hash": "284fe8ace24f0e8a40c03c5b559969abd7caeb29", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/flappy/context_window/train/frame_stack/fixed_l2/fs5/seed_11", + "wandb_unique_id": "flappy-fs5-fixed_l2-s11" +} \ No newline at end of file diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs5:obs30:stride1:seed12/checkpoint_p0/checkpoint_000048840_25034752.pth b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs5:obs30:stride1:seed12/checkpoint_p0/checkpoint_000048840_25034752.pth new file mode 100644 index 0000000000000000000000000000000000000000..b6ef39baddd0521cb24bad019d6f809d92b6376c --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs5:obs30:stride1:seed12/checkpoint_p0/checkpoint_000048840_25034752.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:65f4c745ff7075e4bd24a473c3a8fe1752ee7978d5757c90f854f1979fe1e291 +size 8534589 diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs5:obs30:stride1:seed12/config.json b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs5:obs30:stride1:seed12/config.json new file mode 100644 index 0000000000000000000000000000000000000000..e291805169e8d67115d34682adcae8e2303fcbb8 --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs5:obs30:stride1:seed12/config.json @@ -0,0 +1,266 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_fixed_l2_fs5_seed12", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 12, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs5-fixed_l2", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "fixed", + "fixed_l2", + "fs5", + "seed12" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "screen_size": "", + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "score_limit": -1, + "frame_stack": 5, + "debug": false, + "debug_timelimit_diagnostics": false, + "max_episode_steps": 0, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": null, + "max_latency_ms": null, + "latency_seed": null, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": false, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs5_seed12/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_flappy --experiment flappy_frame_stack_fixed_l2_fs5_seed12 --train_dir results/checkpoints_factor_sweeps/flappy/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-render-batch-size 128 --seed 12 --episode_metrics_path results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs5_seed12/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 2 --num_envs_per_worker 1 --num_policies 1 --batch_size 4096 --rollout 128 --recurrence 1 --num_epochs 8 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 400 --learning_rate 0.00025 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.5 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --use_rnn False --normalize_returns True --normalize_input True --latency-type fixed --fixed-latency-ms 66.66666666666667 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --with_wandb True --wandb_project latency-sensitive-bench --wandb_group flappy-fs5-fixed_l2 --wandb_job_type sample_factory --wandb_tags factor_sweep flappy frame_stack fixed fixed_l2 fs5 seed12 --gym_id FlappyBird-v0 --env-fps 30 --obs-fps 30.0 --use_lidar False --normalize_obs True --audio_on False --obs_resize 84,84 --use-gpu-render True --simulator gpu --gpu-render-device auto --gpu-render-profile False --gpu-render-profile-interval 200 --pipe_gap 100 --bird_color yellow --pipe_color green --background day --frame_stack 5 --debug False --debug-timelimit-diagnostics False --hold-policy one_frame_then_noop --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_fixed_l2_fs5_seed12", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 12, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs5-fixed_l2", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "fixed", + "fixed_l2", + "fs5", + "seed12" + ], + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "frame_stack": 5, + "debug": false, + "debug_timelimit_diagnostics": false, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "add_latency_info": false, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs5_seed12/episode_metrics.jsonl" + }, + "git_hash": "284fe8ace24f0e8a40c03c5b559969abd7caeb29", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/flappy/context_window/train/frame_stack/fixed_l2/fs5/seed_12", + "wandb_unique_id": "flappy-fs5-fixed_l2-s12" +} \ No newline at end of file diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs5:obs30:stride1:seed13/checkpoint_p0/checkpoint_000048816_25034752.pth b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs5:obs30:stride1:seed13/checkpoint_p0/checkpoint_000048816_25034752.pth new file mode 100644 index 0000000000000000000000000000000000000000..73d998aad3e9c3945eff4a4eaee75166a2e5202f --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs5:obs30:stride1:seed13/checkpoint_p0/checkpoint_000048816_25034752.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6576829204e90077b60fed71284fbd5a4efc942004a3abdbec1aea02ceee16af +size 8534589 diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs5:obs30:stride1:seed13/config.json b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs5:obs30:stride1:seed13/config.json new file mode 100644 index 0000000000000000000000000000000000000000..0326bad8d5334847b5659471119bef888c4d9c3c --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs5:obs30:stride1:seed13/config.json @@ -0,0 +1,266 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_fixed_l2_fs5_seed13", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 13, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs5-fixed_l2", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "fixed", + "fixed_l2", + "fs5", + "seed13" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "screen_size": "", + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "score_limit": -1, + "frame_stack": 5, + "debug": false, + "debug_timelimit_diagnostics": false, + "max_episode_steps": 0, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": null, + "max_latency_ms": null, + "latency_seed": null, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": false, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs5_seed13/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_flappy --experiment flappy_frame_stack_fixed_l2_fs5_seed13 --train_dir results/checkpoints_factor_sweeps/flappy/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-render-batch-size 128 --seed 13 --episode_metrics_path results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs5_seed13/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 2 --num_envs_per_worker 1 --num_policies 1 --batch_size 4096 --rollout 128 --recurrence 1 --num_epochs 8 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 400 --learning_rate 0.00025 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.5 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --use_rnn False --normalize_returns True --normalize_input True --latency-type fixed --fixed-latency-ms 66.66666666666667 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --with_wandb True --wandb_project latency-sensitive-bench --wandb_group flappy-fs5-fixed_l2 --wandb_job_type sample_factory --wandb_tags factor_sweep flappy frame_stack fixed fixed_l2 fs5 seed13 --gym_id FlappyBird-v0 --env-fps 30 --obs-fps 30.0 --use_lidar False --normalize_obs True --audio_on False --obs_resize 84,84 --use-gpu-render True --simulator gpu --gpu-render-device auto --gpu-render-profile False --gpu-render-profile-interval 200 --pipe_gap 100 --bird_color yellow --pipe_color green --background day --frame_stack 5 --debug False --debug-timelimit-diagnostics False --hold-policy one_frame_then_noop --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_fixed_l2_fs5_seed13", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 13, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs5-fixed_l2", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "fixed", + "fixed_l2", + "fs5", + "seed13" + ], + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "frame_stack": 5, + "debug": false, + "debug_timelimit_diagnostics": false, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "add_latency_info": false, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs5_seed13/episode_metrics.jsonl" + }, + "git_hash": "284fe8ace24f0e8a40c03c5b559969abd7caeb29", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/flappy/context_window/train/frame_stack/fixed_l2/fs5/seed_13", + "wandb_unique_id": "flappy-fs5-fixed_l2-s13" +} \ No newline at end of file diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs5:obs30:stride1:seed14/checkpoint_p0/checkpoint_000048848_25034752.pth b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs5:obs30:stride1:seed14/checkpoint_p0/checkpoint_000048848_25034752.pth new file mode 100644 index 0000000000000000000000000000000000000000..1af157c55bf2e528c1159afc7c9f1e54d79118c2 --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs5:obs30:stride1:seed14/checkpoint_p0/checkpoint_000048848_25034752.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7fc1da7b2343f2d234cc69b8c36c74a3521facc32b0da149c72552b70afbef3 +size 8534589 diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs5:obs30:stride1:seed14/config.json b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs5:obs30:stride1:seed14/config.json new file mode 100644 index 0000000000000000000000000000000000000000..7162c9f62ff027823ac19e38dde8d141d0a7dc76 --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs5:obs30:stride1:seed14/config.json @@ -0,0 +1,266 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_fixed_l2_fs5_seed14", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 14, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs5-fixed_l2", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "fixed", + "fixed_l2", + "fs5", + "seed14" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "screen_size": "", + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "score_limit": -1, + "frame_stack": 5, + "debug": false, + "debug_timelimit_diagnostics": false, + "max_episode_steps": 0, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": null, + "max_latency_ms": null, + "latency_seed": null, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": false, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs5_seed14/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_flappy --experiment flappy_frame_stack_fixed_l2_fs5_seed14 --train_dir results/checkpoints_factor_sweeps/flappy/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-render-batch-size 128 --seed 14 --episode_metrics_path results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs5_seed14/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 2 --num_envs_per_worker 1 --num_policies 1 --batch_size 4096 --rollout 128 --recurrence 1 --num_epochs 8 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 400 --learning_rate 0.00025 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.5 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --use_rnn False --normalize_returns True --normalize_input True --latency-type fixed --fixed-latency-ms 66.66666666666667 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --with_wandb True --wandb_project latency-sensitive-bench --wandb_group flappy-fs5-fixed_l2 --wandb_job_type sample_factory --wandb_tags factor_sweep flappy frame_stack fixed fixed_l2 fs5 seed14 --gym_id FlappyBird-v0 --env-fps 30 --obs-fps 30.0 --use_lidar False --normalize_obs True --audio_on False --obs_resize 84,84 --use-gpu-render True --simulator gpu --gpu-render-device auto --gpu-render-profile False --gpu-render-profile-interval 200 --pipe_gap 100 --bird_color yellow --pipe_color green --background day --frame_stack 5 --debug False --debug-timelimit-diagnostics False --hold-policy one_frame_then_noop --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_fixed_l2_fs5_seed14", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 14, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs5-fixed_l2", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "fixed", + "fixed_l2", + "fs5", + "seed14" + ], + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "frame_stack": 5, + "debug": false, + "debug_timelimit_diagnostics": false, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "add_latency_info": false, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs5_seed14/episode_metrics.jsonl" + }, + "git_hash": "284fe8ace24f0e8a40c03c5b559969abd7caeb29", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/flappy/context_window/train/frame_stack/fixed_l2/fs5/seed_14", + "wandb_unique_id": "flappy-fs5-fixed_l2-s14" +} \ No newline at end of file diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs6:obs30:stride1:seed10/checkpoint_p0/checkpoint_000048840_25034752.pth b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs6:obs30:stride1:seed10/checkpoint_p0/checkpoint_000048840_25034752.pth new file mode 100644 index 0000000000000000000000000000000000000000..bb012fdb2ea416797ef979c9bb604eb9f83f6465 --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs6:obs30:stride1:seed10/checkpoint_p0/checkpoint_000048840_25034752.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:854b2614f4c9658900ecfb12416fee4135d052623efcc0ea8c575c897eb69559 +size 8897853 diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs6:obs30:stride1:seed10/config.json b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs6:obs30:stride1:seed10/config.json new file mode 100644 index 0000000000000000000000000000000000000000..9a4ba9c4277a16fa8f28b1acd1e83c89f1f0db5f --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs6:obs30:stride1:seed10/config.json @@ -0,0 +1,266 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_fixed_l2_fs6_seed10", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 10, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs6-fixed_l2", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "fixed", + "fixed_l2", + "fs6", + "seed10" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "screen_size": "", + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "score_limit": -1, + "frame_stack": 6, + "debug": false, + "debug_timelimit_diagnostics": false, + "max_episode_steps": 0, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": null, + "max_latency_ms": null, + "latency_seed": null, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": false, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs6_seed10/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_flappy --experiment flappy_frame_stack_fixed_l2_fs6_seed10 --train_dir results/checkpoints_factor_sweeps/flappy/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-render-batch-size 128 --seed 10 --episode_metrics_path results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs6_seed10/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 2 --num_envs_per_worker 1 --num_policies 1 --batch_size 4096 --rollout 128 --recurrence 1 --num_epochs 8 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 400 --learning_rate 0.00025 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.5 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --use_rnn False --normalize_returns True --normalize_input True --latency-type fixed --fixed-latency-ms 66.66666666666667 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --with_wandb True --wandb_project latency-sensitive-bench --wandb_group flappy-fs6-fixed_l2 --wandb_job_type sample_factory --wandb_tags factor_sweep flappy frame_stack fixed fixed_l2 fs6 seed10 --gym_id FlappyBird-v0 --env-fps 30 --obs-fps 30.0 --use_lidar False --normalize_obs True --audio_on False --obs_resize 84,84 --use-gpu-render True --simulator gpu --gpu-render-device auto --gpu-render-profile False --gpu-render-profile-interval 200 --pipe_gap 100 --bird_color yellow --pipe_color green --background day --frame_stack 6 --debug False --debug-timelimit-diagnostics False --hold-policy one_frame_then_noop --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_fixed_l2_fs6_seed10", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 10, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs6-fixed_l2", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "fixed", + "fixed_l2", + "fs6", + "seed10" + ], + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "frame_stack": 6, + "debug": false, + "debug_timelimit_diagnostics": false, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "add_latency_info": false, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs6_seed10/episode_metrics.jsonl" + }, + "git_hash": "284fe8ace24f0e8a40c03c5b559969abd7caeb29", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/flappy/context_window/train/frame_stack/fixed_l2/fs6/seed_10", + "wandb_unique_id": "flappy-fs6-fixed_l2-s10" +} \ No newline at end of file diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs6:obs30:stride1:seed11/checkpoint_p0/checkpoint_000048832_25034752.pth b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs6:obs30:stride1:seed11/checkpoint_p0/checkpoint_000048832_25034752.pth new file mode 100644 index 0000000000000000000000000000000000000000..6c910ff939ddc59dc8217fa80110b24c1eb3357c --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs6:obs30:stride1:seed11/checkpoint_p0/checkpoint_000048832_25034752.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d9ea481336262368596bc7f18c6c4c03453f994586dd6f44dbdf3b79d36baeb5 +size 8897853 diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs6:obs30:stride1:seed11/config.json b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs6:obs30:stride1:seed11/config.json new file mode 100644 index 0000000000000000000000000000000000000000..d4d66bacd8453662d54d31b97cc8191188b2dee2 --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs6:obs30:stride1:seed11/config.json @@ -0,0 +1,266 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_fixed_l2_fs6_seed11", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 11, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs6-fixed_l2", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "fixed", + "fixed_l2", + "fs6", + "seed11" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "screen_size": "", + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "score_limit": -1, + "frame_stack": 6, + "debug": false, + "debug_timelimit_diagnostics": false, + "max_episode_steps": 0, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": null, + "max_latency_ms": null, + "latency_seed": null, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": false, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs6_seed11/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_flappy --experiment flappy_frame_stack_fixed_l2_fs6_seed11 --train_dir results/checkpoints_factor_sweeps/flappy/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-render-batch-size 128 --seed 11 --episode_metrics_path results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs6_seed11/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 2 --num_envs_per_worker 1 --num_policies 1 --batch_size 4096 --rollout 128 --recurrence 1 --num_epochs 8 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 400 --learning_rate 0.00025 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.5 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --use_rnn False --normalize_returns True --normalize_input True --latency-type fixed --fixed-latency-ms 66.66666666666667 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --with_wandb True --wandb_project latency-sensitive-bench --wandb_group flappy-fs6-fixed_l2 --wandb_job_type sample_factory --wandb_tags factor_sweep flappy frame_stack fixed fixed_l2 fs6 seed11 --gym_id FlappyBird-v0 --env-fps 30 --obs-fps 30.0 --use_lidar False --normalize_obs True --audio_on False --obs_resize 84,84 --use-gpu-render True --simulator gpu --gpu-render-device auto --gpu-render-profile False --gpu-render-profile-interval 200 --pipe_gap 100 --bird_color yellow --pipe_color green --background day --frame_stack 6 --debug False --debug-timelimit-diagnostics False --hold-policy one_frame_then_noop --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_fixed_l2_fs6_seed11", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 11, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs6-fixed_l2", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "fixed", + "fixed_l2", + "fs6", + "seed11" + ], + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "frame_stack": 6, + "debug": false, + "debug_timelimit_diagnostics": false, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "add_latency_info": false, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs6_seed11/episode_metrics.jsonl" + }, + "git_hash": "284fe8ace24f0e8a40c03c5b559969abd7caeb29", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/flappy/context_window/train/frame_stack/fixed_l2/fs6/seed_11", + "wandb_unique_id": "flappy-fs6-fixed_l2-s11" +} \ No newline at end of file diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs6:obs30:stride1:seed12/checkpoint_p0/checkpoint_000048832_25034752.pth b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs6:obs30:stride1:seed12/checkpoint_p0/checkpoint_000048832_25034752.pth new file mode 100644 index 0000000000000000000000000000000000000000..47aef4241035f92cf5c5c5d4683e47d35fa637e2 --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs6:obs30:stride1:seed12/checkpoint_p0/checkpoint_000048832_25034752.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:68c266c2ae6f9b109487e107920b5bb6d760042a87636bf365fedcb99670c3e2 +size 8897853 diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs6:obs30:stride1:seed12/config.json b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs6:obs30:stride1:seed12/config.json new file mode 100644 index 0000000000000000000000000000000000000000..430bce897cfd9afc5d7e6b7a87a11a953b6da339 --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs6:obs30:stride1:seed12/config.json @@ -0,0 +1,266 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_fixed_l2_fs6_seed12", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 12, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs6-fixed_l2", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "fixed", + "fixed_l2", + "fs6", + "seed12" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "screen_size": "", + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "score_limit": -1, + "frame_stack": 6, + "debug": false, + "debug_timelimit_diagnostics": false, + "max_episode_steps": 0, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": null, + "max_latency_ms": null, + "latency_seed": null, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": false, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs6_seed12/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_flappy --experiment flappy_frame_stack_fixed_l2_fs6_seed12 --train_dir results/checkpoints_factor_sweeps/flappy/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-render-batch-size 128 --seed 12 --episode_metrics_path results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs6_seed12/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 2 --num_envs_per_worker 1 --num_policies 1 --batch_size 4096 --rollout 128 --recurrence 1 --num_epochs 8 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 400 --learning_rate 0.00025 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.5 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --use_rnn False --normalize_returns True --normalize_input True --latency-type fixed --fixed-latency-ms 66.66666666666667 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --with_wandb True --wandb_project latency-sensitive-bench --wandb_group flappy-fs6-fixed_l2 --wandb_job_type sample_factory --wandb_tags factor_sweep flappy frame_stack fixed fixed_l2 fs6 seed12 --gym_id FlappyBird-v0 --env-fps 30 --obs-fps 30.0 --use_lidar False --normalize_obs True --audio_on False --obs_resize 84,84 --use-gpu-render True --simulator gpu --gpu-render-device auto --gpu-render-profile False --gpu-render-profile-interval 200 --pipe_gap 100 --bird_color yellow --pipe_color green --background day --frame_stack 6 --debug False --debug-timelimit-diagnostics False --hold-policy one_frame_then_noop --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_fixed_l2_fs6_seed12", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 12, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs6-fixed_l2", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "fixed", + "fixed_l2", + "fs6", + "seed12" + ], + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "frame_stack": 6, + "debug": false, + "debug_timelimit_diagnostics": false, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "add_latency_info": false, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs6_seed12/episode_metrics.jsonl" + }, + "git_hash": "284fe8ace24f0e8a40c03c5b559969abd7caeb29", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/flappy/context_window/train/frame_stack/fixed_l2/fs6/seed_12", + "wandb_unique_id": "flappy-fs6-fixed_l2-s12" +} \ No newline at end of file diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs6:obs30:stride1:seed13/checkpoint_p0/checkpoint_000048840_25034752.pth b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs6:obs30:stride1:seed13/checkpoint_p0/checkpoint_000048840_25034752.pth new file mode 100644 index 0000000000000000000000000000000000000000..c1ee951e9771e158d07550a7b8a3cdbbc0d451da --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs6:obs30:stride1:seed13/checkpoint_p0/checkpoint_000048840_25034752.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ab03f8983f8f7f237e266de43a6f2c0a8e12d9d61f4d31a69d735e1eb6f5739f +size 8897853 diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs6:obs30:stride1:seed13/config.json b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs6:obs30:stride1:seed13/config.json new file mode 100644 index 0000000000000000000000000000000000000000..10c760757e7986b6989fe0b115d4fdc238a6b699 --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs6:obs30:stride1:seed13/config.json @@ -0,0 +1,266 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_fixed_l2_fs6_seed13", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 13, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs6-fixed_l2", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "fixed", + "fixed_l2", + "fs6", + "seed13" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "screen_size": "", + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "score_limit": -1, + "frame_stack": 6, + "debug": false, + "debug_timelimit_diagnostics": false, + "max_episode_steps": 0, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": null, + "max_latency_ms": null, + "latency_seed": null, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": false, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs6_seed13/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_flappy --experiment flappy_frame_stack_fixed_l2_fs6_seed13 --train_dir results/checkpoints_factor_sweeps/flappy/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-render-batch-size 128 --seed 13 --episode_metrics_path results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs6_seed13/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 2 --num_envs_per_worker 1 --num_policies 1 --batch_size 4096 --rollout 128 --recurrence 1 --num_epochs 8 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 400 --learning_rate 0.00025 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.5 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --use_rnn False --normalize_returns True --normalize_input True --latency-type fixed --fixed-latency-ms 66.66666666666667 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --with_wandb True --wandb_project latency-sensitive-bench --wandb_group flappy-fs6-fixed_l2 --wandb_job_type sample_factory --wandb_tags factor_sweep flappy frame_stack fixed fixed_l2 fs6 seed13 --gym_id FlappyBird-v0 --env-fps 30 --obs-fps 30.0 --use_lidar False --normalize_obs True --audio_on False --obs_resize 84,84 --use-gpu-render True --simulator gpu --gpu-render-device auto --gpu-render-profile False --gpu-render-profile-interval 200 --pipe_gap 100 --bird_color yellow --pipe_color green --background day --frame_stack 6 --debug False --debug-timelimit-diagnostics False --hold-policy one_frame_then_noop --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_fixed_l2_fs6_seed13", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 13, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs6-fixed_l2", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "fixed", + "fixed_l2", + "fs6", + "seed13" + ], + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "frame_stack": 6, + "debug": false, + "debug_timelimit_diagnostics": false, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "add_latency_info": false, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs6_seed13/episode_metrics.jsonl" + }, + "git_hash": "284fe8ace24f0e8a40c03c5b559969abd7caeb29", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/flappy/context_window/train/frame_stack/fixed_l2/fs6/seed_13", + "wandb_unique_id": "flappy-fs6-fixed_l2-s13" +} \ No newline at end of file diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs6:obs30:stride1:seed14/checkpoint_p0/checkpoint_000048840_25034752.pth b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs6:obs30:stride1:seed14/checkpoint_p0/checkpoint_000048840_25034752.pth new file mode 100644 index 0000000000000000000000000000000000000000..1ba67a65d725810e3981afc1d258ef2bf3d83c68 --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs6:obs30:stride1:seed14/checkpoint_p0/checkpoint_000048840_25034752.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b3012759bbb30b988aa0f562c9fde37eadf352de4077d14fc6737db17b405f06 +size 8897853 diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs6:obs30:stride1:seed14/config.json b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs6:obs30:stride1:seed14/config.json new file mode 100644 index 0000000000000000000000000000000000000000..560c153f2931ebffc76689b95f5ca377de42166b --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs6:obs30:stride1:seed14/config.json @@ -0,0 +1,266 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_fixed_l2_fs6_seed14", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 14, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs6-fixed_l2", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "fixed", + "fixed_l2", + "fs6", + "seed14" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "screen_size": "", + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "score_limit": -1, + "frame_stack": 6, + "debug": false, + "debug_timelimit_diagnostics": false, + "max_episode_steps": 0, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": null, + "max_latency_ms": null, + "latency_seed": null, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": false, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs6_seed14/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_flappy --experiment flappy_frame_stack_fixed_l2_fs6_seed14 --train_dir results/checkpoints_factor_sweeps/flappy/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-render-batch-size 128 --seed 14 --episode_metrics_path results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs6_seed14/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 2 --num_envs_per_worker 1 --num_policies 1 --batch_size 4096 --rollout 128 --recurrence 1 --num_epochs 8 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 400 --learning_rate 0.00025 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.5 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --use_rnn False --normalize_returns True --normalize_input True --latency-type fixed --fixed-latency-ms 66.66666666666667 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --with_wandb True --wandb_project latency-sensitive-bench --wandb_group flappy-fs6-fixed_l2 --wandb_job_type sample_factory --wandb_tags factor_sweep flappy frame_stack fixed fixed_l2 fs6 seed14 --gym_id FlappyBird-v0 --env-fps 30 --obs-fps 30.0 --use_lidar False --normalize_obs True --audio_on False --obs_resize 84,84 --use-gpu-render True --simulator gpu --gpu-render-device auto --gpu-render-profile False --gpu-render-profile-interval 200 --pipe_gap 100 --bird_color yellow --pipe_color green --background day --frame_stack 6 --debug False --debug-timelimit-diagnostics False --hold-policy one_frame_then_noop --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_fixed_l2_fs6_seed14", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 14, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs6-fixed_l2", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "fixed", + "fixed_l2", + "fs6", + "seed14" + ], + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "frame_stack": 6, + "debug": false, + "debug_timelimit_diagnostics": false, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "add_latency_info": false, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs6_seed14/episode_metrics.jsonl" + }, + "git_hash": "284fe8ace24f0e8a40c03c5b559969abd7caeb29", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/flappy/context_window/train/frame_stack/fixed_l2/fs6/seed_14", + "wandb_unique_id": "flappy-fs6-fixed_l2-s14" +} \ No newline at end of file diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs7:obs30:stride1:seed10/checkpoint_p0/checkpoint_000048792_25034752.pth b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs7:obs30:stride1:seed10/checkpoint_p0/checkpoint_000048792_25034752.pth new file mode 100644 index 0000000000000000000000000000000000000000..ef7c31e8c280336e6206dddc0ca22211c78ffdcf --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs7:obs30:stride1:seed10/checkpoint_p0/checkpoint_000048792_25034752.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:56d7ba5083cfedddca740bb949f343fc7d4ff53c88fef5996bf0c5323e897960 +size 9261117 diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs7:obs30:stride1:seed10/config.json b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs7:obs30:stride1:seed10/config.json new file mode 100644 index 0000000000000000000000000000000000000000..4b932e37e7a5acf8c40f287eabc89d2a09cad9f1 --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs7:obs30:stride1:seed10/config.json @@ -0,0 +1,266 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_fixed_l2_fs7_seed10", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 10, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs7-fixed_l2", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "fixed", + "fixed_l2", + "fs7", + "seed10" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "screen_size": "", + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "score_limit": -1, + "frame_stack": 7, + "debug": false, + "debug_timelimit_diagnostics": false, + "max_episode_steps": 0, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": null, + "max_latency_ms": null, + "latency_seed": null, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": false, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs7_seed10/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_flappy --experiment flappy_frame_stack_fixed_l2_fs7_seed10 --train_dir results/checkpoints_factor_sweeps/flappy/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-render-batch-size 128 --seed 10 --episode_metrics_path results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs7_seed10/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 2 --num_envs_per_worker 1 --num_policies 1 --batch_size 4096 --rollout 128 --recurrence 1 --num_epochs 8 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 400 --learning_rate 0.00025 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.5 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --use_rnn False --normalize_returns True --normalize_input True --latency-type fixed --fixed-latency-ms 66.66666666666667 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --with_wandb True --wandb_project latency-sensitive-bench --wandb_group flappy-fs7-fixed_l2 --wandb_job_type sample_factory --wandb_tags factor_sweep flappy frame_stack fixed fixed_l2 fs7 seed10 --gym_id FlappyBird-v0 --env-fps 30 --obs-fps 30.0 --use_lidar False --normalize_obs True --audio_on False --obs_resize 84,84 --use-gpu-render True --simulator gpu --gpu-render-device auto --gpu-render-profile False --gpu-render-profile-interval 200 --pipe_gap 100 --bird_color yellow --pipe_color green --background day --frame_stack 7 --debug False --debug-timelimit-diagnostics False --hold-policy one_frame_then_noop --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_fixed_l2_fs7_seed10", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 10, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs7-fixed_l2", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "fixed", + "fixed_l2", + "fs7", + "seed10" + ], + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "frame_stack": 7, + "debug": false, + "debug_timelimit_diagnostics": false, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "add_latency_info": false, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs7_seed10/episode_metrics.jsonl" + }, + "git_hash": "284fe8ace24f0e8a40c03c5b559969abd7caeb29", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/flappy/context_window/train/frame_stack/fixed_l2/fs7/seed_10", + "wandb_unique_id": "flappy-fs7-fixed_l2-s10" +} \ No newline at end of file diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs7:obs30:stride1:seed11/checkpoint_p0/checkpoint_000048832_25034752.pth b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs7:obs30:stride1:seed11/checkpoint_p0/checkpoint_000048832_25034752.pth new file mode 100644 index 0000000000000000000000000000000000000000..847a97d6438762eb0ab93ad7ecd3f44d712d1367 --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs7:obs30:stride1:seed11/checkpoint_p0/checkpoint_000048832_25034752.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1a8c57152120a76f76fdf0ff8340fbcf9e70be5bcdaafddc2c19abaa7686d052 +size 9261117 diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs7:obs30:stride1:seed11/config.json b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs7:obs30:stride1:seed11/config.json new file mode 100644 index 0000000000000000000000000000000000000000..eb39e62170593d9ef5a04fabc879a5ab44ede150 --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs7:obs30:stride1:seed11/config.json @@ -0,0 +1,266 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_fixed_l2_fs7_seed11", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 11, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs7-fixed_l2", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "fixed", + "fixed_l2", + "fs7", + "seed11" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "screen_size": "", + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "score_limit": -1, + "frame_stack": 7, + "debug": false, + "debug_timelimit_diagnostics": false, + "max_episode_steps": 0, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": null, + "max_latency_ms": null, + "latency_seed": null, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": false, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs7_seed11/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_flappy --experiment flappy_frame_stack_fixed_l2_fs7_seed11 --train_dir results/checkpoints_factor_sweeps/flappy/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-render-batch-size 128 --seed 11 --episode_metrics_path results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs7_seed11/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 2 --num_envs_per_worker 1 --num_policies 1 --batch_size 4096 --rollout 128 --recurrence 1 --num_epochs 8 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 400 --learning_rate 0.00025 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.5 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --use_rnn False --normalize_returns True --normalize_input True --latency-type fixed --fixed-latency-ms 66.66666666666667 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --with_wandb True --wandb_project latency-sensitive-bench --wandb_group flappy-fs7-fixed_l2 --wandb_job_type sample_factory --wandb_tags factor_sweep flappy frame_stack fixed fixed_l2 fs7 seed11 --gym_id FlappyBird-v0 --env-fps 30 --obs-fps 30.0 --use_lidar False --normalize_obs True --audio_on False --obs_resize 84,84 --use-gpu-render True --simulator gpu --gpu-render-device auto --gpu-render-profile False --gpu-render-profile-interval 200 --pipe_gap 100 --bird_color yellow --pipe_color green --background day --frame_stack 7 --debug False --debug-timelimit-diagnostics False --hold-policy one_frame_then_noop --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_fixed_l2_fs7_seed11", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 11, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs7-fixed_l2", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "fixed", + "fixed_l2", + "fs7", + "seed11" + ], + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "frame_stack": 7, + "debug": false, + "debug_timelimit_diagnostics": false, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "add_latency_info": false, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs7_seed11/episode_metrics.jsonl" + }, + "git_hash": "284fe8ace24f0e8a40c03c5b559969abd7caeb29", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/flappy/context_window/train/frame_stack/fixed_l2/fs7/seed_11", + "wandb_unique_id": "flappy-fs7-fixed_l2-s11" +} \ No newline at end of file diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs7:obs30:stride1:seed12/checkpoint_p0/checkpoint_000048840_25034752.pth b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs7:obs30:stride1:seed12/checkpoint_p0/checkpoint_000048840_25034752.pth new file mode 100644 index 0000000000000000000000000000000000000000..66b60640645663e0caff2e66f4f2b4ae0d3d9aaf --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs7:obs30:stride1:seed12/checkpoint_p0/checkpoint_000048840_25034752.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f642f32a351295db4e9430812462b66a735d1b53436be10e253b1600baa23fe9 +size 9261117 diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs7:obs30:stride1:seed12/config.json b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs7:obs30:stride1:seed12/config.json new file mode 100644 index 0000000000000000000000000000000000000000..3aad67fd0705d6a1f659b6532d19c55add46c736 --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs7:obs30:stride1:seed12/config.json @@ -0,0 +1,266 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_fixed_l2_fs7_seed12", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 12, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs7-fixed_l2", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "fixed", + "fixed_l2", + "fs7", + "seed12" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "screen_size": "", + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "score_limit": -1, + "frame_stack": 7, + "debug": false, + "debug_timelimit_diagnostics": false, + "max_episode_steps": 0, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": null, + "max_latency_ms": null, + "latency_seed": null, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": false, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs7_seed12/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_flappy --experiment flappy_frame_stack_fixed_l2_fs7_seed12 --train_dir results/checkpoints_factor_sweeps/flappy/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-render-batch-size 128 --seed 12 --episode_metrics_path results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs7_seed12/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 2 --num_envs_per_worker 1 --num_policies 1 --batch_size 4096 --rollout 128 --recurrence 1 --num_epochs 8 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 400 --learning_rate 0.00025 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.5 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --use_rnn False --normalize_returns True --normalize_input True --latency-type fixed --fixed-latency-ms 66.66666666666667 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --with_wandb True --wandb_project latency-sensitive-bench --wandb_group flappy-fs7-fixed_l2 --wandb_job_type sample_factory --wandb_tags factor_sweep flappy frame_stack fixed fixed_l2 fs7 seed12 --gym_id FlappyBird-v0 --env-fps 30 --obs-fps 30.0 --use_lidar False --normalize_obs True --audio_on False --obs_resize 84,84 --use-gpu-render True --simulator gpu --gpu-render-device auto --gpu-render-profile False --gpu-render-profile-interval 200 --pipe_gap 100 --bird_color yellow --pipe_color green --background day --frame_stack 7 --debug False --debug-timelimit-diagnostics False --hold-policy one_frame_then_noop --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_fixed_l2_fs7_seed12", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 12, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs7-fixed_l2", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "fixed", + "fixed_l2", + "fs7", + "seed12" + ], + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "frame_stack": 7, + "debug": false, + "debug_timelimit_diagnostics": false, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "add_latency_info": false, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs7_seed12/episode_metrics.jsonl" + }, + "git_hash": "284fe8ace24f0e8a40c03c5b559969abd7caeb29", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/flappy/context_window/train/frame_stack/fixed_l2/fs7/seed_12", + "wandb_unique_id": "flappy-fs7-fixed_l2-s12" +} \ No newline at end of file diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs7:obs30:stride1:seed13/checkpoint_p0/checkpoint_000048840_25034752.pth b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs7:obs30:stride1:seed13/checkpoint_p0/checkpoint_000048840_25034752.pth new file mode 100644 index 0000000000000000000000000000000000000000..db16c8e9b3971581d92c06b9e6c4840639bd2350 --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs7:obs30:stride1:seed13/checkpoint_p0/checkpoint_000048840_25034752.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7572975bcde630753022b8a7f17cee9ae6e172abcb0ff278bd961c8074e1ce75 +size 9261117 diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs7:obs30:stride1:seed13/config.json b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs7:obs30:stride1:seed13/config.json new file mode 100644 index 0000000000000000000000000000000000000000..786e122266879073390db7ec2d423758d28a9e33 --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs7:obs30:stride1:seed13/config.json @@ -0,0 +1,266 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_fixed_l2_fs7_seed13", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 13, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs7-fixed_l2", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "fixed", + "fixed_l2", + "fs7", + "seed13" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "screen_size": "", + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "score_limit": -1, + "frame_stack": 7, + "debug": false, + "debug_timelimit_diagnostics": false, + "max_episode_steps": 0, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": null, + "max_latency_ms": null, + "latency_seed": null, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": false, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs7_seed13/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_flappy --experiment flappy_frame_stack_fixed_l2_fs7_seed13 --train_dir results/checkpoints_factor_sweeps/flappy/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-render-batch-size 128 --seed 13 --episode_metrics_path results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs7_seed13/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 2 --num_envs_per_worker 1 --num_policies 1 --batch_size 4096 --rollout 128 --recurrence 1 --num_epochs 8 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 400 --learning_rate 0.00025 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.5 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --use_rnn False --normalize_returns True --normalize_input True --latency-type fixed --fixed-latency-ms 66.66666666666667 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --with_wandb True --wandb_project latency-sensitive-bench --wandb_group flappy-fs7-fixed_l2 --wandb_job_type sample_factory --wandb_tags factor_sweep flappy frame_stack fixed fixed_l2 fs7 seed13 --gym_id FlappyBird-v0 --env-fps 30 --obs-fps 30.0 --use_lidar False --normalize_obs True --audio_on False --obs_resize 84,84 --use-gpu-render True --simulator gpu --gpu-render-device auto --gpu-render-profile False --gpu-render-profile-interval 200 --pipe_gap 100 --bird_color yellow --pipe_color green --background day --frame_stack 7 --debug False --debug-timelimit-diagnostics False --hold-policy one_frame_then_noop --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_fixed_l2_fs7_seed13", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 13, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs7-fixed_l2", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "fixed", + "fixed_l2", + "fs7", + "seed13" + ], + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "frame_stack": 7, + "debug": false, + "debug_timelimit_diagnostics": false, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "add_latency_info": false, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs7_seed13/episode_metrics.jsonl" + }, + "git_hash": "284fe8ace24f0e8a40c03c5b559969abd7caeb29", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/flappy/context_window/train/frame_stack/fixed_l2/fs7/seed_13", + "wandb_unique_id": "flappy-fs7-fixed_l2-s13" +} \ No newline at end of file diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs7:obs30:stride1:seed14/checkpoint_p0/checkpoint_000048784_25034752.pth b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs7:obs30:stride1:seed14/checkpoint_p0/checkpoint_000048784_25034752.pth new file mode 100644 index 0000000000000000000000000000000000000000..f75dd09fef0908b3234443d39204c241c98493ae --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs7:obs30:stride1:seed14/checkpoint_p0/checkpoint_000048784_25034752.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:22f1a6b1e6fd7d799fbfe474d455dc3f41391a3c295f6dfc74bc8d7d158917aa +size 9261117 diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs7:obs30:stride1:seed14/config.json b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs7:obs30:stride1:seed14/config.json new file mode 100644 index 0000000000000000000000000000000000000000..9154161eb9921de39fdb5b74980c65504c68d843 --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs7:obs30:stride1:seed14/config.json @@ -0,0 +1,266 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_fixed_l2_fs7_seed14", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 14, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs7-fixed_l2", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "fixed", + "fixed_l2", + "fs7", + "seed14" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "screen_size": "", + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "score_limit": -1, + "frame_stack": 7, + "debug": false, + "debug_timelimit_diagnostics": false, + "max_episode_steps": 0, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": null, + "max_latency_ms": null, + "latency_seed": null, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": false, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs7_seed14/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_flappy --experiment flappy_frame_stack_fixed_l2_fs7_seed14 --train_dir results/checkpoints_factor_sweeps/flappy/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-render-batch-size 128 --seed 14 --episode_metrics_path results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs7_seed14/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 2 --num_envs_per_worker 1 --num_policies 1 --batch_size 4096 --rollout 128 --recurrence 1 --num_epochs 8 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 400 --learning_rate 0.00025 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.5 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --use_rnn False --normalize_returns True --normalize_input True --latency-type fixed --fixed-latency-ms 66.66666666666667 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --with_wandb True --wandb_project latency-sensitive-bench --wandb_group flappy-fs7-fixed_l2 --wandb_job_type sample_factory --wandb_tags factor_sweep flappy frame_stack fixed fixed_l2 fs7 seed14 --gym_id FlappyBird-v0 --env-fps 30 --obs-fps 30.0 --use_lidar False --normalize_obs True --audio_on False --obs_resize 84,84 --use-gpu-render True --simulator gpu --gpu-render-device auto --gpu-render-profile False --gpu-render-profile-interval 200 --pipe_gap 100 --bird_color yellow --pipe_color green --background day --frame_stack 7 --debug False --debug-timelimit-diagnostics False --hold-policy one_frame_then_noop --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_fixed_l2_fs7_seed14", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 14, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs7-fixed_l2", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "fixed", + "fixed_l2", + "fs7", + "seed14" + ], + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "frame_stack": 7, + "debug": false, + "debug_timelimit_diagnostics": false, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "add_latency_info": false, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs7_seed14/episode_metrics.jsonl" + }, + "git_hash": "284fe8ace24f0e8a40c03c5b559969abd7caeb29", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/flappy/context_window/train/frame_stack/fixed_l2/fs7/seed_14", + "wandb_unique_id": "flappy-fs7-fixed_l2-s14" +} \ No newline at end of file diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs8:obs30:stride1:seed10/checkpoint_p0/checkpoint_000048824_25034752.pth b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs8:obs30:stride1:seed10/checkpoint_p0/checkpoint_000048824_25034752.pth new file mode 100644 index 0000000000000000000000000000000000000000..aff5a2ab0fa19eb3dbd60de4a3204e438c5ef345 --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs8:obs30:stride1:seed10/checkpoint_p0/checkpoint_000048824_25034752.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:db08817efeecb3cdfda4072dfc22e7be0c513c70f6dfab5e7d5c5e7825445f5b +size 9624381 diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs8:obs30:stride1:seed10/config.json b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs8:obs30:stride1:seed10/config.json new file mode 100644 index 0000000000000000000000000000000000000000..80fd5b27818c57684f07853ab94689a135627cba --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs8:obs30:stride1:seed10/config.json @@ -0,0 +1,266 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_fixed_l2_fs8_seed10", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 10, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs8-fixed_l2", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "fixed", + "fixed_l2", + "fs8", + "seed10" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "screen_size": "", + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "score_limit": -1, + "frame_stack": 8, + "debug": false, + "debug_timelimit_diagnostics": false, + "max_episode_steps": 0, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": null, + "max_latency_ms": null, + "latency_seed": null, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": false, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs8_seed10/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_flappy --experiment flappy_frame_stack_fixed_l2_fs8_seed10 --train_dir results/checkpoints_factor_sweeps/flappy/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-render-batch-size 128 --seed 10 --episode_metrics_path results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs8_seed10/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 2 --num_envs_per_worker 1 --num_policies 1 --batch_size 4096 --rollout 128 --recurrence 1 --num_epochs 8 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 400 --learning_rate 0.00025 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.5 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --use_rnn False --normalize_returns True --normalize_input True --latency-type fixed --fixed-latency-ms 66.66666666666667 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --with_wandb True --wandb_project latency-sensitive-bench --wandb_group flappy-fs8-fixed_l2 --wandb_job_type sample_factory --wandb_tags factor_sweep flappy frame_stack fixed fixed_l2 fs8 seed10 --gym_id FlappyBird-v0 --env-fps 30 --obs-fps 30.0 --use_lidar False --normalize_obs True --audio_on False --obs_resize 84,84 --use-gpu-render True --simulator gpu --gpu-render-device auto --gpu-render-profile False --gpu-render-profile-interval 200 --pipe_gap 100 --bird_color yellow --pipe_color green --background day --frame_stack 8 --debug False --debug-timelimit-diagnostics False --hold-policy one_frame_then_noop --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_fixed_l2_fs8_seed10", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 10, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs8-fixed_l2", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "fixed", + "fixed_l2", + "fs8", + "seed10" + ], + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "frame_stack": 8, + "debug": false, + "debug_timelimit_diagnostics": false, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "add_latency_info": false, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs8_seed10/episode_metrics.jsonl" + }, + "git_hash": "284fe8ace24f0e8a40c03c5b559969abd7caeb29", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/flappy/context_window/train/frame_stack/fixed_l2/fs8/seed_10", + "wandb_unique_id": "flappy-fs8-fixed_l2-s10" +} \ No newline at end of file diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs8:obs30:stride1:seed11/checkpoint_p0/checkpoint_000048832_25034752.pth b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs8:obs30:stride1:seed11/checkpoint_p0/checkpoint_000048832_25034752.pth new file mode 100644 index 0000000000000000000000000000000000000000..5c89e03a6490b0df8bddeb49ad85499858b4ecce --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs8:obs30:stride1:seed11/checkpoint_p0/checkpoint_000048832_25034752.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b51fa3c3b33a685ef58910798147f2420e745f559a3dcf9c58c05aec64dcd9e6 +size 9624381 diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs8:obs30:stride1:seed11/config.json b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs8:obs30:stride1:seed11/config.json new file mode 100644 index 0000000000000000000000000000000000000000..5cc57b59b20938bba0d8dad80894d904cefed918 --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs8:obs30:stride1:seed11/config.json @@ -0,0 +1,266 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_fixed_l2_fs8_seed11", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 11, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs8-fixed_l2", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "fixed", + "fixed_l2", + "fs8", + "seed11" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "screen_size": "", + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "score_limit": -1, + "frame_stack": 8, + "debug": false, + "debug_timelimit_diagnostics": false, + "max_episode_steps": 0, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": null, + "max_latency_ms": null, + "latency_seed": null, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": false, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs8_seed11/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_flappy --experiment flappy_frame_stack_fixed_l2_fs8_seed11 --train_dir results/checkpoints_factor_sweeps/flappy/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-render-batch-size 128 --seed 11 --episode_metrics_path results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs8_seed11/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 2 --num_envs_per_worker 1 --num_policies 1 --batch_size 4096 --rollout 128 --recurrence 1 --num_epochs 8 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 400 --learning_rate 0.00025 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.5 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --use_rnn False --normalize_returns True --normalize_input True --latency-type fixed --fixed-latency-ms 66.66666666666667 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --with_wandb True --wandb_project latency-sensitive-bench --wandb_group flappy-fs8-fixed_l2 --wandb_job_type sample_factory --wandb_tags factor_sweep flappy frame_stack fixed fixed_l2 fs8 seed11 --gym_id FlappyBird-v0 --env-fps 30 --obs-fps 30.0 --use_lidar False --normalize_obs True --audio_on False --obs_resize 84,84 --use-gpu-render True --simulator gpu --gpu-render-device auto --gpu-render-profile False --gpu-render-profile-interval 200 --pipe_gap 100 --bird_color yellow --pipe_color green --background day --frame_stack 8 --debug False --debug-timelimit-diagnostics False --hold-policy one_frame_then_noop --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_fixed_l2_fs8_seed11", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 11, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs8-fixed_l2", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "fixed", + "fixed_l2", + "fs8", + "seed11" + ], + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "frame_stack": 8, + "debug": false, + "debug_timelimit_diagnostics": false, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "add_latency_info": false, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs8_seed11/episode_metrics.jsonl" + }, + "git_hash": "284fe8ace24f0e8a40c03c5b559969abd7caeb29", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/flappy/context_window/train/frame_stack/fixed_l2/fs8/seed_11", + "wandb_unique_id": "flappy-fs8-fixed_l2-s11" +} \ No newline at end of file diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs8:obs30:stride1:seed12/checkpoint_p0/checkpoint_000048832_25034752.pth b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs8:obs30:stride1:seed12/checkpoint_p0/checkpoint_000048832_25034752.pth new file mode 100644 index 0000000000000000000000000000000000000000..ef089efdcc2901098c6930b3d52ab5d45eea7090 --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs8:obs30:stride1:seed12/checkpoint_p0/checkpoint_000048832_25034752.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:184d34465562a7a4a24e454f9a685817079fd4c386a0f757bc89520194bb5e3a +size 9624381 diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs8:obs30:stride1:seed12/config.json b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs8:obs30:stride1:seed12/config.json new file mode 100644 index 0000000000000000000000000000000000000000..78694e298703437c955b8fb036756482fe8ab73c --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs8:obs30:stride1:seed12/config.json @@ -0,0 +1,266 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_fixed_l2_fs8_seed12", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 12, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs8-fixed_l2", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "fixed", + "fixed_l2", + "fs8", + "seed12" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "screen_size": "", + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "score_limit": -1, + "frame_stack": 8, + "debug": false, + "debug_timelimit_diagnostics": false, + "max_episode_steps": 0, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": null, + "max_latency_ms": null, + "latency_seed": null, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": false, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs8_seed12/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_flappy --experiment flappy_frame_stack_fixed_l2_fs8_seed12 --train_dir results/checkpoints_factor_sweeps/flappy/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-render-batch-size 128 --seed 12 --episode_metrics_path results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs8_seed12/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 2 --num_envs_per_worker 1 --num_policies 1 --batch_size 4096 --rollout 128 --recurrence 1 --num_epochs 8 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 400 --learning_rate 0.00025 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.5 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --use_rnn False --normalize_returns True --normalize_input True --latency-type fixed --fixed-latency-ms 66.66666666666667 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --with_wandb True --wandb_project latency-sensitive-bench --wandb_group flappy-fs8-fixed_l2 --wandb_job_type sample_factory --wandb_tags factor_sweep flappy frame_stack fixed fixed_l2 fs8 seed12 --gym_id FlappyBird-v0 --env-fps 30 --obs-fps 30.0 --use_lidar False --normalize_obs True --audio_on False --obs_resize 84,84 --use-gpu-render True --simulator gpu --gpu-render-device auto --gpu-render-profile False --gpu-render-profile-interval 200 --pipe_gap 100 --bird_color yellow --pipe_color green --background day --frame_stack 8 --debug False --debug-timelimit-diagnostics False --hold-policy one_frame_then_noop --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_fixed_l2_fs8_seed12", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 12, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs8-fixed_l2", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "fixed", + "fixed_l2", + "fs8", + "seed12" + ], + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "frame_stack": 8, + "debug": false, + "debug_timelimit_diagnostics": false, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "add_latency_info": false, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs8_seed12/episode_metrics.jsonl" + }, + "git_hash": "284fe8ace24f0e8a40c03c5b559969abd7caeb29", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/flappy/context_window/train/frame_stack/fixed_l2/fs8/seed_12", + "wandb_unique_id": "flappy-fs8-fixed_l2-s12" +} \ No newline at end of file diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs8:obs30:stride1:seed13/checkpoint_p0/checkpoint_000048832_25034752.pth b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs8:obs30:stride1:seed13/checkpoint_p0/checkpoint_000048832_25034752.pth new file mode 100644 index 0000000000000000000000000000000000000000..1b582bfa034d8c0885748dd546f963786cdcd386 --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs8:obs30:stride1:seed13/checkpoint_p0/checkpoint_000048832_25034752.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c12440d612b2e6b240eb587d572dd2438b3177298115a86ca3deed307a9aa0fc +size 9624381 diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs8:obs30:stride1:seed13/config.json b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs8:obs30:stride1:seed13/config.json new file mode 100644 index 0000000000000000000000000000000000000000..361526f1f7cb10a486ac93d178045164c573e7c6 --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs8:obs30:stride1:seed13/config.json @@ -0,0 +1,266 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_fixed_l2_fs8_seed13", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 13, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs8-fixed_l2", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "fixed", + "fixed_l2", + "fs8", + "seed13" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "screen_size": "", + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "score_limit": -1, + "frame_stack": 8, + "debug": false, + "debug_timelimit_diagnostics": false, + "max_episode_steps": 0, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": null, + "max_latency_ms": null, + "latency_seed": null, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": false, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs8_seed13/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_flappy --experiment flappy_frame_stack_fixed_l2_fs8_seed13 --train_dir results/checkpoints_factor_sweeps/flappy/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-render-batch-size 128 --seed 13 --episode_metrics_path results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs8_seed13/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 2 --num_envs_per_worker 1 --num_policies 1 --batch_size 4096 --rollout 128 --recurrence 1 --num_epochs 8 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 400 --learning_rate 0.00025 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.5 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --use_rnn False --normalize_returns True --normalize_input True --latency-type fixed --fixed-latency-ms 66.66666666666667 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --with_wandb True --wandb_project latency-sensitive-bench --wandb_group flappy-fs8-fixed_l2 --wandb_job_type sample_factory --wandb_tags factor_sweep flappy frame_stack fixed fixed_l2 fs8 seed13 --gym_id FlappyBird-v0 --env-fps 30 --obs-fps 30.0 --use_lidar False --normalize_obs True --audio_on False --obs_resize 84,84 --use-gpu-render True --simulator gpu --gpu-render-device auto --gpu-render-profile False --gpu-render-profile-interval 200 --pipe_gap 100 --bird_color yellow --pipe_color green --background day --frame_stack 8 --debug False --debug-timelimit-diagnostics False --hold-policy one_frame_then_noop --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_fixed_l2_fs8_seed13", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 13, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs8-fixed_l2", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "fixed", + "fixed_l2", + "fs8", + "seed13" + ], + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "frame_stack": 8, + "debug": false, + "debug_timelimit_diagnostics": false, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "add_latency_info": false, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs8_seed13/episode_metrics.jsonl" + }, + "git_hash": "284fe8ace24f0e8a40c03c5b559969abd7caeb29", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/flappy/context_window/train/frame_stack/fixed_l2/fs8/seed_13", + "wandb_unique_id": "flappy-fs8-fixed_l2-s13" +} \ No newline at end of file diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs8:obs30:stride1:seed14/checkpoint_p0/checkpoint_000048840_25034752.pth b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs8:obs30:stride1:seed14/checkpoint_p0/checkpoint_000048840_25034752.pth new file mode 100644 index 0000000000000000000000000000000000000000..05a1d85630ac45e68f02017763194da072441851 --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs8:obs30:stride1:seed14/checkpoint_p0/checkpoint_000048840_25034752.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ad123c26c2bc4f04bbb5bf3a61f0c4fe64b5bb8f4237ccbe03e5fd0bf491c2bc +size 9624381 diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs8:obs30:stride1:seed14/config.json b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs8:obs30:stride1:seed14/config.json new file mode 100644 index 0000000000000000000000000000000000000000..8a224aa1cd489ce9c541d2b1bc80600412de85ee --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:fixed_l2:fs8:obs30:stride1:seed14/config.json @@ -0,0 +1,266 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_fixed_l2_fs8_seed14", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 14, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs8-fixed_l2", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "fixed", + "fixed_l2", + "fs8", + "seed14" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "screen_size": "", + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "score_limit": -1, + "frame_stack": 8, + "debug": false, + "debug_timelimit_diagnostics": false, + "max_episode_steps": 0, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": null, + "max_latency_ms": null, + "latency_seed": null, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": false, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs8_seed14/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_flappy --experiment flappy_frame_stack_fixed_l2_fs8_seed14 --train_dir results/checkpoints_factor_sweeps/flappy/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-render-batch-size 128 --seed 14 --episode_metrics_path results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs8_seed14/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 2 --num_envs_per_worker 1 --num_policies 1 --batch_size 4096 --rollout 128 --recurrence 1 --num_epochs 8 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 400 --learning_rate 0.00025 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.5 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --use_rnn False --normalize_returns True --normalize_input True --latency-type fixed --fixed-latency-ms 66.66666666666667 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --with_wandb True --wandb_project latency-sensitive-bench --wandb_group flappy-fs8-fixed_l2 --wandb_job_type sample_factory --wandb_tags factor_sweep flappy frame_stack fixed fixed_l2 fs8 seed14 --gym_id FlappyBird-v0 --env-fps 30 --obs-fps 30.0 --use_lidar False --normalize_obs True --audio_on False --obs_resize 84,84 --use-gpu-render True --simulator gpu --gpu-render-device auto --gpu-render-profile False --gpu-render-profile-interval 200 --pipe_gap 100 --bird_color yellow --pipe_color green --background day --frame_stack 8 --debug False --debug-timelimit-diagnostics False --hold-policy one_frame_then_noop --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_fixed_l2_fs8_seed14", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 14, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs8-fixed_l2", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "fixed", + "fixed_l2", + "fs8", + "seed14" + ], + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "frame_stack": 8, + "debug": false, + "debug_timelimit_diagnostics": false, + "mode": "train", + "latency_type": "fixed", + "fixed_latency_ms": 66.66666666666667, + "add_latency_info": false, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_fixed_l2_fs8_seed14/episode_metrics.jsonl" + }, + "git_hash": "284fe8ace24f0e8a40c03c5b559969abd7caeb29", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/flappy/context_window/train/frame_stack/fixed_l2/fs8/seed_14", + "wandb_unique_id": "flappy-fs8-fixed_l2-s14" +} \ No newline at end of file diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs1:obs30:stride1:seed10/checkpoint_p0/checkpoint_000048808_25034752.pth b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs1:obs30:stride1:seed10/checkpoint_p0/checkpoint_000048808_25034752.pth new file mode 100644 index 0000000000000000000000000000000000000000..6f7d53eb4a14ae941e8e0c69448b4ead87cbd1bc --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs1:obs30:stride1:seed10/checkpoint_p0/checkpoint_000048808_25034752.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2455452ae98c92b8480837cfbd03bc6541f7936d7ff309511622822a28e4a64c +size 7081533 diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs1:obs30:stride1:seed10/config.json b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs1:obs30:stride1:seed10/config.json new file mode 100644 index 0000000000000000000000000000000000000000..347b2919729cc505edc8a3e64b795e4ca621cb54 --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs1:obs30:stride1:seed10/config.json @@ -0,0 +1,268 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_uniform_u1_3_fs1_seed10", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 10, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs1-uniform_u1_3", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "uniform", + "uniform_u1_3", + "fs1", + "seed10" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "screen_size": "", + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "score_limit": -1, + "frame_stack": 1, + "debug": false, + "debug_timelimit_diagnostics": false, + "max_episode_steps": 0, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 10, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": false, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_uniform_u1_3_fs1_seed10/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_flappy --experiment flappy_frame_stack_uniform_u1_3_fs1_seed10 --train_dir results/checkpoints_factor_sweeps/flappy/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-render-batch-size 128 --seed 10 --episode_metrics_path results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_uniform_u1_3_fs1_seed10/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 2 --num_envs_per_worker 1 --num_policies 1 --batch_size 4096 --rollout 128 --recurrence 1 --num_epochs 8 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 400 --learning_rate 0.00025 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.5 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 10 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --with_wandb True --wandb_project latency-sensitive-bench --wandb_group flappy-fs1-uniform_u1_3 --wandb_job_type sample_factory --wandb_tags factor_sweep flappy frame_stack uniform uniform_u1_3 fs1 seed10 --gym_id FlappyBird-v0 --env-fps 30 --obs-fps 30.0 --use_lidar False --normalize_obs True --audio_on False --obs_resize 84,84 --use-gpu-render True --simulator gpu --gpu-render-device auto --gpu-render-profile False --gpu-render-profile-interval 200 --pipe_gap 100 --bird_color yellow --pipe_color green --background day --frame_stack 1 --debug False --debug-timelimit-diagnostics False --hold-policy one_frame_then_noop --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_uniform_u1_3_fs1_seed10", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 10, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs1-uniform_u1_3", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "uniform", + "uniform_u1_3", + "fs1", + "seed10" + ], + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "frame_stack": 1, + "debug": false, + "debug_timelimit_diagnostics": false, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 10, + "add_latency_info": false, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_uniform_u1_3_fs1_seed10/episode_metrics.jsonl" + }, + "git_hash": "eb3a2e1efbd2aa03a60d7f44f5e18d8fdd0f5a2d", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/flappy/context_window/train/frame_stack/uniform_u1_3/fs1/seed_10", + "wandb_unique_id": "flappy-fs1-uniform_u1_3-s10" +} \ No newline at end of file diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs1:obs30:stride1:seed11/checkpoint_p0/checkpoint_000048832_25034752.pth b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs1:obs30:stride1:seed11/checkpoint_p0/checkpoint_000048832_25034752.pth new file mode 100644 index 0000000000000000000000000000000000000000..a0210d163bf94ca9b4fddeec7ceb08f95ece739e --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs1:obs30:stride1:seed11/checkpoint_p0/checkpoint_000048832_25034752.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9eba7448b7d10f43093f954dcd33b6ac5a91aa8c74ac22abcb9ea4780b2b9893 +size 7081533 diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs1:obs30:stride1:seed11/config.json b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs1:obs30:stride1:seed11/config.json new file mode 100644 index 0000000000000000000000000000000000000000..04eae622a33cb5556c751feecb769a558e6ed891 --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs1:obs30:stride1:seed11/config.json @@ -0,0 +1,268 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_uniform_u1_3_fs1_seed11", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 11, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs1-uniform_u1_3", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "uniform", + "uniform_u1_3", + "fs1", + "seed11" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "screen_size": "", + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "score_limit": -1, + "frame_stack": 1, + "debug": false, + "debug_timelimit_diagnostics": false, + "max_episode_steps": 0, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 11, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": false, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_uniform_u1_3_fs1_seed11/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_flappy --experiment flappy_frame_stack_uniform_u1_3_fs1_seed11 --train_dir results/checkpoints_factor_sweeps/flappy/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-render-batch-size 128 --seed 11 --episode_metrics_path results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_uniform_u1_3_fs1_seed11/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 2 --num_envs_per_worker 1 --num_policies 1 --batch_size 4096 --rollout 128 --recurrence 1 --num_epochs 8 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 400 --learning_rate 0.00025 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.5 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 11 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --with_wandb True --wandb_project latency-sensitive-bench --wandb_group flappy-fs1-uniform_u1_3 --wandb_job_type sample_factory --wandb_tags factor_sweep flappy frame_stack uniform uniform_u1_3 fs1 seed11 --gym_id FlappyBird-v0 --env-fps 30 --obs-fps 30.0 --use_lidar False --normalize_obs True --audio_on False --obs_resize 84,84 --use-gpu-render True --simulator gpu --gpu-render-device auto --gpu-render-profile False --gpu-render-profile-interval 200 --pipe_gap 100 --bird_color yellow --pipe_color green --background day --frame_stack 1 --debug False --debug-timelimit-diagnostics False --hold-policy one_frame_then_noop --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_uniform_u1_3_fs1_seed11", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 11, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs1-uniform_u1_3", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "uniform", + "uniform_u1_3", + "fs1", + "seed11" + ], + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "frame_stack": 1, + "debug": false, + "debug_timelimit_diagnostics": false, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 11, + "add_latency_info": false, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_uniform_u1_3_fs1_seed11/episode_metrics.jsonl" + }, + "git_hash": "eb3a2e1efbd2aa03a60d7f44f5e18d8fdd0f5a2d", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/flappy/context_window/train/frame_stack/uniform_u1_3/fs1/seed_11", + "wandb_unique_id": "flappy-fs1-uniform_u1_3-s11" +} \ No newline at end of file diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs1:obs30:stride1:seed12/checkpoint_p0/checkpoint_000048832_25034752.pth b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs1:obs30:stride1:seed12/checkpoint_p0/checkpoint_000048832_25034752.pth new file mode 100644 index 0000000000000000000000000000000000000000..cb77a2c4e3e4877686ec62016d13661348797a96 --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs1:obs30:stride1:seed12/checkpoint_p0/checkpoint_000048832_25034752.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fd16e74560611650a5e7291e695eadce5029abf82f81a30c4aba589e521a3448 +size 7081533 diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs1:obs30:stride1:seed12/config.json b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs1:obs30:stride1:seed12/config.json new file mode 100644 index 0000000000000000000000000000000000000000..7847c0fe1f8622a94d33c2df9adbde4964488c06 --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs1:obs30:stride1:seed12/config.json @@ -0,0 +1,268 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_uniform_u1_3_fs1_seed12", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 12, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs1-uniform_u1_3", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "uniform", + "uniform_u1_3", + "fs1", + "seed12" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "screen_size": "", + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "score_limit": -1, + "frame_stack": 1, + "debug": false, + "debug_timelimit_diagnostics": false, + "max_episode_steps": 0, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 12, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": false, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_uniform_u1_3_fs1_seed12/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_flappy --experiment flappy_frame_stack_uniform_u1_3_fs1_seed12 --train_dir results/checkpoints_factor_sweeps/flappy/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-render-batch-size 128 --seed 12 --episode_metrics_path results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_uniform_u1_3_fs1_seed12/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 2 --num_envs_per_worker 1 --num_policies 1 --batch_size 4096 --rollout 128 --recurrence 1 --num_epochs 8 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 400 --learning_rate 0.00025 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.5 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 12 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --with_wandb True --wandb_project latency-sensitive-bench --wandb_group flappy-fs1-uniform_u1_3 --wandb_job_type sample_factory --wandb_tags factor_sweep flappy frame_stack uniform uniform_u1_3 fs1 seed12 --gym_id FlappyBird-v0 --env-fps 30 --obs-fps 30.0 --use_lidar False --normalize_obs True --audio_on False --obs_resize 84,84 --use-gpu-render True --simulator gpu --gpu-render-device auto --gpu-render-profile False --gpu-render-profile-interval 200 --pipe_gap 100 --bird_color yellow --pipe_color green --background day --frame_stack 1 --debug False --debug-timelimit-diagnostics False --hold-policy one_frame_then_noop --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_uniform_u1_3_fs1_seed12", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 12, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs1-uniform_u1_3", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "uniform", + "uniform_u1_3", + "fs1", + "seed12" + ], + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "frame_stack": 1, + "debug": false, + "debug_timelimit_diagnostics": false, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 12, + "add_latency_info": false, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_uniform_u1_3_fs1_seed12/episode_metrics.jsonl" + }, + "git_hash": "eb3a2e1efbd2aa03a60d7f44f5e18d8fdd0f5a2d", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/flappy/context_window/train/frame_stack/uniform_u1_3/fs1/seed_12", + "wandb_unique_id": "flappy-fs1-uniform_u1_3-s12" +} \ No newline at end of file diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs1:obs30:stride1:seed13/checkpoint_p0/checkpoint_000048664_25034752.pth b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs1:obs30:stride1:seed13/checkpoint_p0/checkpoint_000048664_25034752.pth new file mode 100644 index 0000000000000000000000000000000000000000..40cfe4fc14a6c3f5e79f0a73d01b03b07098d976 --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs1:obs30:stride1:seed13/checkpoint_p0/checkpoint_000048664_25034752.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:22fa51c7683581eae805e28d275eb95c08a48835d3151ecf15bd956fb2a8e07b +size 7081533 diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs1:obs30:stride1:seed13/config.json b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs1:obs30:stride1:seed13/config.json new file mode 100644 index 0000000000000000000000000000000000000000..a02f19cbe3f474ecc598abae3070c7fbbfdb1a79 --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs1:obs30:stride1:seed13/config.json @@ -0,0 +1,268 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_uniform_u1_3_fs1_seed13", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 13, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs1-uniform_u1_3", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "uniform", + "uniform_u1_3", + "fs1", + "seed13" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "screen_size": "", + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "score_limit": -1, + "frame_stack": 1, + "debug": false, + "debug_timelimit_diagnostics": false, + "max_episode_steps": 0, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 13, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": false, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_uniform_u1_3_fs1_seed13/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_flappy --experiment flappy_frame_stack_uniform_u1_3_fs1_seed13 --train_dir results/checkpoints_factor_sweeps/flappy/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-render-batch-size 128 --seed 13 --episode_metrics_path results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_uniform_u1_3_fs1_seed13/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 2 --num_envs_per_worker 1 --num_policies 1 --batch_size 4096 --rollout 128 --recurrence 1 --num_epochs 8 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 400 --learning_rate 0.00025 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.5 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 13 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --with_wandb True --wandb_project latency-sensitive-bench --wandb_group flappy-fs1-uniform_u1_3 --wandb_job_type sample_factory --wandb_tags factor_sweep flappy frame_stack uniform uniform_u1_3 fs1 seed13 --gym_id FlappyBird-v0 --env-fps 30 --obs-fps 30.0 --use_lidar False --normalize_obs True --audio_on False --obs_resize 84,84 --use-gpu-render True --simulator gpu --gpu-render-device auto --gpu-render-profile False --gpu-render-profile-interval 200 --pipe_gap 100 --bird_color yellow --pipe_color green --background day --frame_stack 1 --debug False --debug-timelimit-diagnostics False --hold-policy one_frame_then_noop --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_uniform_u1_3_fs1_seed13", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 13, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs1-uniform_u1_3", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "uniform", + "uniform_u1_3", + "fs1", + "seed13" + ], + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "frame_stack": 1, + "debug": false, + "debug_timelimit_diagnostics": false, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 13, + "add_latency_info": false, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_uniform_u1_3_fs1_seed13/episode_metrics.jsonl" + }, + "git_hash": "eb3a2e1efbd2aa03a60d7f44f5e18d8fdd0f5a2d", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/flappy/context_window/train/frame_stack/uniform_u1_3/fs1/seed_13", + "wandb_unique_id": "flappy-fs1-uniform_u1_3-s13" +} \ No newline at end of file diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs1:obs30:stride1:seed14/checkpoint_p0/checkpoint_000048840_25034752.pth b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs1:obs30:stride1:seed14/checkpoint_p0/checkpoint_000048840_25034752.pth new file mode 100644 index 0000000000000000000000000000000000000000..9e2e160a33e21ba2fd7909017e9dc67c6e362c60 --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs1:obs30:stride1:seed14/checkpoint_p0/checkpoint_000048840_25034752.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d90cef254e79cea01e09188f70db7fef2220b032bf456da85bd56e5339ce3025 +size 7081533 diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs1:obs30:stride1:seed14/config.json b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs1:obs30:stride1:seed14/config.json new file mode 100644 index 0000000000000000000000000000000000000000..51749453a3b04687dc9b4e7ba67f55f4d733e1a0 --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs1:obs30:stride1:seed14/config.json @@ -0,0 +1,268 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_uniform_u1_3_fs1_seed14", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 14, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs1-uniform_u1_3", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "uniform", + "uniform_u1_3", + "fs1", + "seed14" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "screen_size": "", + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "score_limit": -1, + "frame_stack": 1, + "debug": false, + "debug_timelimit_diagnostics": false, + "max_episode_steps": 0, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 14, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": false, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_uniform_u1_3_fs1_seed14/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_flappy --experiment flappy_frame_stack_uniform_u1_3_fs1_seed14 --train_dir results/checkpoints_factor_sweeps/flappy/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-render-batch-size 128 --seed 14 --episode_metrics_path results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_uniform_u1_3_fs1_seed14/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 2 --num_envs_per_worker 1 --num_policies 1 --batch_size 4096 --rollout 128 --recurrence 1 --num_epochs 8 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 400 --learning_rate 0.00025 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.5 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 14 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --with_wandb True --wandb_project latency-sensitive-bench --wandb_group flappy-fs1-uniform_u1_3 --wandb_job_type sample_factory --wandb_tags factor_sweep flappy frame_stack uniform uniform_u1_3 fs1 seed14 --gym_id FlappyBird-v0 --env-fps 30 --obs-fps 30.0 --use_lidar False --normalize_obs True --audio_on False --obs_resize 84,84 --use-gpu-render True --simulator gpu --gpu-render-device auto --gpu-render-profile False --gpu-render-profile-interval 200 --pipe_gap 100 --bird_color yellow --pipe_color green --background day --frame_stack 1 --debug False --debug-timelimit-diagnostics False --hold-policy one_frame_then_noop --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_uniform_u1_3_fs1_seed14", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 14, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs1-uniform_u1_3", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "uniform", + "uniform_u1_3", + "fs1", + "seed14" + ], + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "frame_stack": 1, + "debug": false, + "debug_timelimit_diagnostics": false, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 14, + "add_latency_info": false, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_uniform_u1_3_fs1_seed14/episode_metrics.jsonl" + }, + "git_hash": "eb3a2e1efbd2aa03a60d7f44f5e18d8fdd0f5a2d", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/flappy/context_window/train/frame_stack/uniform_u1_3/fs1/seed_14", + "wandb_unique_id": "flappy-fs1-uniform_u1_3-s14" +} \ No newline at end of file diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs2:obs30:stride1:seed10/checkpoint_p0/checkpoint_000048840_25034752.pth b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs2:obs30:stride1:seed10/checkpoint_p0/checkpoint_000048840_25034752.pth new file mode 100644 index 0000000000000000000000000000000000000000..6d8194915a1e380c8a19e9f5c8d2334b9724542e --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs2:obs30:stride1:seed10/checkpoint_p0/checkpoint_000048840_25034752.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d4c0ad1f7aa03b8e8d4e51982c3efd34eed9eb19c8b5ec62ae1e8bbb4dc11024 +size 7444797 diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs2:obs30:stride1:seed10/config.json b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs2:obs30:stride1:seed10/config.json new file mode 100644 index 0000000000000000000000000000000000000000..36f78b462d2ce64b723ceaef0eb16752a7914039 --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs2:obs30:stride1:seed10/config.json @@ -0,0 +1,268 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_uniform_u1_3_fs2_seed10", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 10, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs2-uniform_u1_3", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "uniform", + "uniform_u1_3", + "fs2", + "seed10" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "screen_size": "", + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "score_limit": -1, + "frame_stack": 2, + "debug": false, + "debug_timelimit_diagnostics": false, + "max_episode_steps": 0, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 10, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": false, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_uniform_u1_3_fs2_seed10/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_flappy --experiment flappy_frame_stack_uniform_u1_3_fs2_seed10 --train_dir results/checkpoints_factor_sweeps/flappy/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-render-batch-size 128 --seed 10 --episode_metrics_path results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_uniform_u1_3_fs2_seed10/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 2 --num_envs_per_worker 1 --num_policies 1 --batch_size 4096 --rollout 128 --recurrence 1 --num_epochs 8 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 400 --learning_rate 0.00025 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.5 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 10 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --with_wandb True --wandb_project latency-sensitive-bench --wandb_group flappy-fs2-uniform_u1_3 --wandb_job_type sample_factory --wandb_tags factor_sweep flappy frame_stack uniform uniform_u1_3 fs2 seed10 --gym_id FlappyBird-v0 --env-fps 30 --obs-fps 30.0 --use_lidar False --normalize_obs True --audio_on False --obs_resize 84,84 --use-gpu-render True --simulator gpu --gpu-render-device auto --gpu-render-profile False --gpu-render-profile-interval 200 --pipe_gap 100 --bird_color yellow --pipe_color green --background day --frame_stack 2 --debug False --debug-timelimit-diagnostics False --hold-policy one_frame_then_noop --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_uniform_u1_3_fs2_seed10", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 10, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs2-uniform_u1_3", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "uniform", + "uniform_u1_3", + "fs2", + "seed10" + ], + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "frame_stack": 2, + "debug": false, + "debug_timelimit_diagnostics": false, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 10, + "add_latency_info": false, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_uniform_u1_3_fs2_seed10/episode_metrics.jsonl" + }, + "git_hash": "eb3a2e1efbd2aa03a60d7f44f5e18d8fdd0f5a2d", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/flappy/context_window/train/frame_stack/uniform_u1_3/fs2/seed_10", + "wandb_unique_id": "flappy-fs2-uniform_u1_3-s10" +} \ No newline at end of file diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs2:obs30:stride1:seed11/checkpoint_p0/checkpoint_000048840_25034752.pth b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs2:obs30:stride1:seed11/checkpoint_p0/checkpoint_000048840_25034752.pth new file mode 100644 index 0000000000000000000000000000000000000000..7374b2acee059e0e9ee81e57fe78521a75725f7a --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs2:obs30:stride1:seed11/checkpoint_p0/checkpoint_000048840_25034752.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:27cef91a8ebfec30d5db58f7c423c17f55af63532b861db1fa0f5c0660458980 +size 7444797 diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs2:obs30:stride1:seed11/config.json b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs2:obs30:stride1:seed11/config.json new file mode 100644 index 0000000000000000000000000000000000000000..05cfecb0d7aaa92c156af830d032a8cd531f61e4 --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs2:obs30:stride1:seed11/config.json @@ -0,0 +1,268 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_uniform_u1_3_fs2_seed11", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 11, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs2-uniform_u1_3", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "uniform", + "uniform_u1_3", + "fs2", + "seed11" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "screen_size": "", + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "score_limit": -1, + "frame_stack": 2, + "debug": false, + "debug_timelimit_diagnostics": false, + "max_episode_steps": 0, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 11, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": false, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_uniform_u1_3_fs2_seed11/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_flappy --experiment flappy_frame_stack_uniform_u1_3_fs2_seed11 --train_dir results/checkpoints_factor_sweeps/flappy/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-render-batch-size 128 --seed 11 --episode_metrics_path results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_uniform_u1_3_fs2_seed11/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 2 --num_envs_per_worker 1 --num_policies 1 --batch_size 4096 --rollout 128 --recurrence 1 --num_epochs 8 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 400 --learning_rate 0.00025 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.5 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 11 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --with_wandb True --wandb_project latency-sensitive-bench --wandb_group flappy-fs2-uniform_u1_3 --wandb_job_type sample_factory --wandb_tags factor_sweep flappy frame_stack uniform uniform_u1_3 fs2 seed11 --gym_id FlappyBird-v0 --env-fps 30 --obs-fps 30.0 --use_lidar False --normalize_obs True --audio_on False --obs_resize 84,84 --use-gpu-render True --simulator gpu --gpu-render-device auto --gpu-render-profile False --gpu-render-profile-interval 200 --pipe_gap 100 --bird_color yellow --pipe_color green --background day --frame_stack 2 --debug False --debug-timelimit-diagnostics False --hold-policy one_frame_then_noop --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_uniform_u1_3_fs2_seed11", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 11, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs2-uniform_u1_3", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "uniform", + "uniform_u1_3", + "fs2", + "seed11" + ], + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "frame_stack": 2, + "debug": false, + "debug_timelimit_diagnostics": false, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 11, + "add_latency_info": false, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_uniform_u1_3_fs2_seed11/episode_metrics.jsonl" + }, + "git_hash": "eb3a2e1efbd2aa03a60d7f44f5e18d8fdd0f5a2d", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/flappy/context_window/train/frame_stack/uniform_u1_3/fs2/seed_11", + "wandb_unique_id": "flappy-fs2-uniform_u1_3-s11" +} \ No newline at end of file diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs2:obs30:stride1:seed12/checkpoint_p0/checkpoint_000048768_25034752.pth b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs2:obs30:stride1:seed12/checkpoint_p0/checkpoint_000048768_25034752.pth new file mode 100644 index 0000000000000000000000000000000000000000..8a30992ad8ecb2ca782f6154755d0b17e19aa9f3 --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs2:obs30:stride1:seed12/checkpoint_p0/checkpoint_000048768_25034752.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8522e93dd4b763de134be2551293df5eed84651680de018f4c5783d417709bca +size 7444797 diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs2:obs30:stride1:seed12/config.json b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs2:obs30:stride1:seed12/config.json new file mode 100644 index 0000000000000000000000000000000000000000..4fd06fbd3777f089b2e74ac0e90a13c790e0fdda --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs2:obs30:stride1:seed12/config.json @@ -0,0 +1,268 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_uniform_u1_3_fs2_seed12", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 12, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs2-uniform_u1_3", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "uniform", + "uniform_u1_3", + "fs2", + "seed12" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "screen_size": "", + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "score_limit": -1, + "frame_stack": 2, + "debug": false, + "debug_timelimit_diagnostics": false, + "max_episode_steps": 0, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 12, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": false, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_uniform_u1_3_fs2_seed12/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_flappy --experiment flappy_frame_stack_uniform_u1_3_fs2_seed12 --train_dir results/checkpoints_factor_sweeps/flappy/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-render-batch-size 128 --seed 12 --episode_metrics_path results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_uniform_u1_3_fs2_seed12/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 2 --num_envs_per_worker 1 --num_policies 1 --batch_size 4096 --rollout 128 --recurrence 1 --num_epochs 8 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 400 --learning_rate 0.00025 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.5 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 12 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --with_wandb True --wandb_project latency-sensitive-bench --wandb_group flappy-fs2-uniform_u1_3 --wandb_job_type sample_factory --wandb_tags factor_sweep flappy frame_stack uniform uniform_u1_3 fs2 seed12 --gym_id FlappyBird-v0 --env-fps 30 --obs-fps 30.0 --use_lidar False --normalize_obs True --audio_on False --obs_resize 84,84 --use-gpu-render True --simulator gpu --gpu-render-device auto --gpu-render-profile False --gpu-render-profile-interval 200 --pipe_gap 100 --bird_color yellow --pipe_color green --background day --frame_stack 2 --debug False --debug-timelimit-diagnostics False --hold-policy one_frame_then_noop --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_uniform_u1_3_fs2_seed12", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 12, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs2-uniform_u1_3", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "uniform", + "uniform_u1_3", + "fs2", + "seed12" + ], + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "frame_stack": 2, + "debug": false, + "debug_timelimit_diagnostics": false, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 12, + "add_latency_info": false, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_uniform_u1_3_fs2_seed12/episode_metrics.jsonl" + }, + "git_hash": "eb3a2e1efbd2aa03a60d7f44f5e18d8fdd0f5a2d", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/flappy/context_window/train/frame_stack/uniform_u1_3/fs2/seed_12", + "wandb_unique_id": "flappy-fs2-uniform_u1_3-s12" +} \ No newline at end of file diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs2:obs30:stride1:seed13/checkpoint_p0/checkpoint_000048840_25034752.pth b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs2:obs30:stride1:seed13/checkpoint_p0/checkpoint_000048840_25034752.pth new file mode 100644 index 0000000000000000000000000000000000000000..a65c30dc44295b521978c214fe171fa7945b6d6c --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs2:obs30:stride1:seed13/checkpoint_p0/checkpoint_000048840_25034752.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e69d07213f3365378b054511c38e41cf8b1737ca32a6354aef3256e073b94e1a +size 7444797 diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs2:obs30:stride1:seed13/config.json b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs2:obs30:stride1:seed13/config.json new file mode 100644 index 0000000000000000000000000000000000000000..adcff3a74fbc89ed69ffd726fe5ff9e8c00773c7 --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs2:obs30:stride1:seed13/config.json @@ -0,0 +1,268 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_uniform_u1_3_fs2_seed13", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 13, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs2-uniform_u1_3", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "uniform", + "uniform_u1_3", + "fs2", + "seed13" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "screen_size": "", + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "score_limit": -1, + "frame_stack": 2, + "debug": false, + "debug_timelimit_diagnostics": false, + "max_episode_steps": 0, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 13, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": false, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_uniform_u1_3_fs2_seed13/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_flappy --experiment flappy_frame_stack_uniform_u1_3_fs2_seed13 --train_dir results/checkpoints_factor_sweeps/flappy/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-render-batch-size 128 --seed 13 --episode_metrics_path results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_uniform_u1_3_fs2_seed13/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 2 --num_envs_per_worker 1 --num_policies 1 --batch_size 4096 --rollout 128 --recurrence 1 --num_epochs 8 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 400 --learning_rate 0.00025 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.5 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 13 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --with_wandb True --wandb_project latency-sensitive-bench --wandb_group flappy-fs2-uniform_u1_3 --wandb_job_type sample_factory --wandb_tags factor_sweep flappy frame_stack uniform uniform_u1_3 fs2 seed13 --gym_id FlappyBird-v0 --env-fps 30 --obs-fps 30.0 --use_lidar False --normalize_obs True --audio_on False --obs_resize 84,84 --use-gpu-render True --simulator gpu --gpu-render-device auto --gpu-render-profile False --gpu-render-profile-interval 200 --pipe_gap 100 --bird_color yellow --pipe_color green --background day --frame_stack 2 --debug False --debug-timelimit-diagnostics False --hold-policy one_frame_then_noop --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_uniform_u1_3_fs2_seed13", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 13, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs2-uniform_u1_3", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "uniform", + "uniform_u1_3", + "fs2", + "seed13" + ], + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "frame_stack": 2, + "debug": false, + "debug_timelimit_diagnostics": false, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 13, + "add_latency_info": false, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_uniform_u1_3_fs2_seed13/episode_metrics.jsonl" + }, + "git_hash": "eb3a2e1efbd2aa03a60d7f44f5e18d8fdd0f5a2d", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/flappy/context_window/train/frame_stack/uniform_u1_3/fs2/seed_13", + "wandb_unique_id": "flappy-fs2-uniform_u1_3-s13" +} \ No newline at end of file diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs2:obs30:stride1:seed14/checkpoint_p0/checkpoint_000048752_25034752.pth b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs2:obs30:stride1:seed14/checkpoint_p0/checkpoint_000048752_25034752.pth new file mode 100644 index 0000000000000000000000000000000000000000..7844b5cc80bfd6527f8d65df454c851c9ea13ebb --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs2:obs30:stride1:seed14/checkpoint_p0/checkpoint_000048752_25034752.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1a9ccceaa89876208a9ae682fbbe92703721cd0e3120dd36c99d257a4d04283c +size 7444797 diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs2:obs30:stride1:seed14/config.json b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs2:obs30:stride1:seed14/config.json new file mode 100644 index 0000000000000000000000000000000000000000..0a0abe4405f73325a67d78c1807d05395101f550 --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs2:obs30:stride1:seed14/config.json @@ -0,0 +1,268 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_uniform_u1_3_fs2_seed14", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 14, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs2-uniform_u1_3", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "uniform", + "uniform_u1_3", + "fs2", + "seed14" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "screen_size": "", + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "score_limit": -1, + "frame_stack": 2, + "debug": false, + "debug_timelimit_diagnostics": false, + "max_episode_steps": 0, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 14, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": false, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_uniform_u1_3_fs2_seed14/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_flappy --experiment flappy_frame_stack_uniform_u1_3_fs2_seed14 --train_dir results/checkpoints_factor_sweeps/flappy/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-render-batch-size 128 --seed 14 --episode_metrics_path results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_uniform_u1_3_fs2_seed14/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 2 --num_envs_per_worker 1 --num_policies 1 --batch_size 4096 --rollout 128 --recurrence 1 --num_epochs 8 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 400 --learning_rate 0.00025 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.5 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 14 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --with_wandb True --wandb_project latency-sensitive-bench --wandb_group flappy-fs2-uniform_u1_3 --wandb_job_type sample_factory --wandb_tags factor_sweep flappy frame_stack uniform uniform_u1_3 fs2 seed14 --gym_id FlappyBird-v0 --env-fps 30 --obs-fps 30.0 --use_lidar False --normalize_obs True --audio_on False --obs_resize 84,84 --use-gpu-render True --simulator gpu --gpu-render-device auto --gpu-render-profile False --gpu-render-profile-interval 200 --pipe_gap 100 --bird_color yellow --pipe_color green --background day --frame_stack 2 --debug False --debug-timelimit-diagnostics False --hold-policy one_frame_then_noop --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_uniform_u1_3_fs2_seed14", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 14, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs2-uniform_u1_3", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "uniform", + "uniform_u1_3", + "fs2", + "seed14" + ], + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "frame_stack": 2, + "debug": false, + "debug_timelimit_diagnostics": false, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 14, + "add_latency_info": false, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_uniform_u1_3_fs2_seed14/episode_metrics.jsonl" + }, + "git_hash": "eb3a2e1efbd2aa03a60d7f44f5e18d8fdd0f5a2d", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/flappy/context_window/train/frame_stack/uniform_u1_3/fs2/seed_14", + "wandb_unique_id": "flappy-fs2-uniform_u1_3-s14" +} \ No newline at end of file diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs3:obs30:stride1:seed10/checkpoint_p0/checkpoint_000048824_25034752.pth b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs3:obs30:stride1:seed10/checkpoint_p0/checkpoint_000048824_25034752.pth new file mode 100644 index 0000000000000000000000000000000000000000..0f0699fd89acb5f2b2b31c69e3d6415a99ae5bc4 --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs3:obs30:stride1:seed10/checkpoint_p0/checkpoint_000048824_25034752.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:27cca26cd729cce895b780235cd3cdf1a89fd478cc0433932fe2f058d4fb0535 +size 7808061 diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs3:obs30:stride1:seed10/config.json b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs3:obs30:stride1:seed10/config.json new file mode 100644 index 0000000000000000000000000000000000000000..898b5612707a2d82127157db75a51c3d3142454c --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs3:obs30:stride1:seed10/config.json @@ -0,0 +1,268 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_uniform_u1_3_fs3_seed10", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 10, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs3-uniform_u1_3", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "uniform", + "uniform_u1_3", + "fs3", + "seed10" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "screen_size": "", + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "score_limit": -1, + "frame_stack": 3, + "debug": false, + "debug_timelimit_diagnostics": false, + "max_episode_steps": 0, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 10, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": false, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_uniform_u1_3_fs3_seed10/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_flappy --experiment flappy_frame_stack_uniform_u1_3_fs3_seed10 --train_dir results/checkpoints_factor_sweeps/flappy/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-render-batch-size 128 --seed 10 --episode_metrics_path results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_uniform_u1_3_fs3_seed10/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 2 --num_envs_per_worker 1 --num_policies 1 --batch_size 4096 --rollout 128 --recurrence 1 --num_epochs 8 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 400 --learning_rate 0.00025 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.5 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 10 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --with_wandb True --wandb_project latency-sensitive-bench --wandb_group flappy-fs3-uniform_u1_3 --wandb_job_type sample_factory --wandb_tags factor_sweep flappy frame_stack uniform uniform_u1_3 fs3 seed10 --gym_id FlappyBird-v0 --env-fps 30 --obs-fps 30.0 --use_lidar False --normalize_obs True --audio_on False --obs_resize 84,84 --use-gpu-render True --simulator gpu --gpu-render-device auto --gpu-render-profile False --gpu-render-profile-interval 200 --pipe_gap 100 --bird_color yellow --pipe_color green --background day --frame_stack 3 --debug False --debug-timelimit-diagnostics False --hold-policy one_frame_then_noop --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_uniform_u1_3_fs3_seed10", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 10, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs3-uniform_u1_3", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "uniform", + "uniform_u1_3", + "fs3", + "seed10" + ], + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "frame_stack": 3, + "debug": false, + "debug_timelimit_diagnostics": false, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 10, + "add_latency_info": false, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_uniform_u1_3_fs3_seed10/episode_metrics.jsonl" + }, + "git_hash": "eb3a2e1efbd2aa03a60d7f44f5e18d8fdd0f5a2d", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/flappy/context_window/train/frame_stack/uniform_u1_3/fs3/seed_10", + "wandb_unique_id": "flappy-fs3-uniform_u1_3-s10" +} \ No newline at end of file diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs3:obs30:stride1:seed11/checkpoint_p0/checkpoint_000048816_25034752.pth b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs3:obs30:stride1:seed11/checkpoint_p0/checkpoint_000048816_25034752.pth new file mode 100644 index 0000000000000000000000000000000000000000..725cad48ef5c043bd60f9732e9b1492412813035 --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs3:obs30:stride1:seed11/checkpoint_p0/checkpoint_000048816_25034752.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e5819aed2b3bbb75c8e80eddd8409e258f5e1ce2142932396ef511600d17002b +size 7808061 diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs3:obs30:stride1:seed11/config.json b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs3:obs30:stride1:seed11/config.json new file mode 100644 index 0000000000000000000000000000000000000000..64c9b0d350d46b6d8eeb80c815a96d02cdf44512 --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs3:obs30:stride1:seed11/config.json @@ -0,0 +1,268 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_uniform_u1_3_fs3_seed11", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 11, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs3-uniform_u1_3", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "uniform", + "uniform_u1_3", + "fs3", + "seed11" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "screen_size": "", + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "score_limit": -1, + "frame_stack": 3, + "debug": false, + "debug_timelimit_diagnostics": false, + "max_episode_steps": 0, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 11, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": false, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_uniform_u1_3_fs3_seed11/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_flappy --experiment flappy_frame_stack_uniform_u1_3_fs3_seed11 --train_dir results/checkpoints_factor_sweeps/flappy/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-render-batch-size 128 --seed 11 --episode_metrics_path results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_uniform_u1_3_fs3_seed11/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 2 --num_envs_per_worker 1 --num_policies 1 --batch_size 4096 --rollout 128 --recurrence 1 --num_epochs 8 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 400 --learning_rate 0.00025 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.5 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 11 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --with_wandb True --wandb_project latency-sensitive-bench --wandb_group flappy-fs3-uniform_u1_3 --wandb_job_type sample_factory --wandb_tags factor_sweep flappy frame_stack uniform uniform_u1_3 fs3 seed11 --gym_id FlappyBird-v0 --env-fps 30 --obs-fps 30.0 --use_lidar False --normalize_obs True --audio_on False --obs_resize 84,84 --use-gpu-render True --simulator gpu --gpu-render-device auto --gpu-render-profile False --gpu-render-profile-interval 200 --pipe_gap 100 --bird_color yellow --pipe_color green --background day --frame_stack 3 --debug False --debug-timelimit-diagnostics False --hold-policy one_frame_then_noop --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_uniform_u1_3_fs3_seed11", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 11, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs3-uniform_u1_3", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "uniform", + "uniform_u1_3", + "fs3", + "seed11" + ], + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "frame_stack": 3, + "debug": false, + "debug_timelimit_diagnostics": false, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 11, + "add_latency_info": false, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_uniform_u1_3_fs3_seed11/episode_metrics.jsonl" + }, + "git_hash": "eb3a2e1efbd2aa03a60d7f44f5e18d8fdd0f5a2d", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/flappy/context_window/train/frame_stack/uniform_u1_3/fs3/seed_11", + "wandb_unique_id": "flappy-fs3-uniform_u1_3-s11" +} \ No newline at end of file diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs3:obs30:stride1:seed12/checkpoint_p0/checkpoint_000048832_25034752.pth b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs3:obs30:stride1:seed12/checkpoint_p0/checkpoint_000048832_25034752.pth new file mode 100644 index 0000000000000000000000000000000000000000..e8bc4cd243127d0ac53f385b591cc082c66d3b1b --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs3:obs30:stride1:seed12/checkpoint_p0/checkpoint_000048832_25034752.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:905a984545856bff45302653a02fc1022d9577845687f7083d7347f9740f6e21 +size 7808061 diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs3:obs30:stride1:seed12/config.json b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs3:obs30:stride1:seed12/config.json new file mode 100644 index 0000000000000000000000000000000000000000..04190798b654498c929930bf9d2e42eee4d91967 --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs3:obs30:stride1:seed12/config.json @@ -0,0 +1,268 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_uniform_u1_3_fs3_seed12", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 12, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs3-uniform_u1_3", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "uniform", + "uniform_u1_3", + "fs3", + "seed12" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "screen_size": "", + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "score_limit": -1, + "frame_stack": 3, + "debug": false, + "debug_timelimit_diagnostics": false, + "max_episode_steps": 0, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 12, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": false, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_uniform_u1_3_fs3_seed12/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_flappy --experiment flappy_frame_stack_uniform_u1_3_fs3_seed12 --train_dir results/checkpoints_factor_sweeps/flappy/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-render-batch-size 128 --seed 12 --episode_metrics_path results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_uniform_u1_3_fs3_seed12/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 2 --num_envs_per_worker 1 --num_policies 1 --batch_size 4096 --rollout 128 --recurrence 1 --num_epochs 8 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 400 --learning_rate 0.00025 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.5 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 12 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --with_wandb True --wandb_project latency-sensitive-bench --wandb_group flappy-fs3-uniform_u1_3 --wandb_job_type sample_factory --wandb_tags factor_sweep flappy frame_stack uniform uniform_u1_3 fs3 seed12 --gym_id FlappyBird-v0 --env-fps 30 --obs-fps 30.0 --use_lidar False --normalize_obs True --audio_on False --obs_resize 84,84 --use-gpu-render True --simulator gpu --gpu-render-device auto --gpu-render-profile False --gpu-render-profile-interval 200 --pipe_gap 100 --bird_color yellow --pipe_color green --background day --frame_stack 3 --debug False --debug-timelimit-diagnostics False --hold-policy one_frame_then_noop --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_uniform_u1_3_fs3_seed12", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 12, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs3-uniform_u1_3", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "uniform", + "uniform_u1_3", + "fs3", + "seed12" + ], + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "frame_stack": 3, + "debug": false, + "debug_timelimit_diagnostics": false, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 12, + "add_latency_info": false, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_uniform_u1_3_fs3_seed12/episode_metrics.jsonl" + }, + "git_hash": "eb3a2e1efbd2aa03a60d7f44f5e18d8fdd0f5a2d", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/flappy/context_window/train/frame_stack/uniform_u1_3/fs3/seed_12", + "wandb_unique_id": "flappy-fs3-uniform_u1_3-s12" +} \ No newline at end of file diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs3:obs30:stride1:seed13/checkpoint_p0/checkpoint_000048848_25034752.pth b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs3:obs30:stride1:seed13/checkpoint_p0/checkpoint_000048848_25034752.pth new file mode 100644 index 0000000000000000000000000000000000000000..216ee0c7d5cd97d0fb0b6bb7b7d50b714ae882b8 --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs3:obs30:stride1:seed13/checkpoint_p0/checkpoint_000048848_25034752.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:72408e365cfbbc12e3a0968c358d8725302f294d49301f192e2021c4528e700b +size 7808061 diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs3:obs30:stride1:seed13/config.json b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs3:obs30:stride1:seed13/config.json new file mode 100644 index 0000000000000000000000000000000000000000..740b475b6fd4e3062c3989d43968773a6cab6e0f --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs3:obs30:stride1:seed13/config.json @@ -0,0 +1,268 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_uniform_u1_3_fs3_seed13", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 13, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs3-uniform_u1_3", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "uniform", + "uniform_u1_3", + "fs3", + "seed13" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "screen_size": "", + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "score_limit": -1, + "frame_stack": 3, + "debug": false, + "debug_timelimit_diagnostics": false, + "max_episode_steps": 0, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 13, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": false, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_uniform_u1_3_fs3_seed13/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_flappy --experiment flappy_frame_stack_uniform_u1_3_fs3_seed13 --train_dir results/checkpoints_factor_sweeps/flappy/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-render-batch-size 128 --seed 13 --episode_metrics_path results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_uniform_u1_3_fs3_seed13/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 2 --num_envs_per_worker 1 --num_policies 1 --batch_size 4096 --rollout 128 --recurrence 1 --num_epochs 8 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 400 --learning_rate 0.00025 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.5 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 13 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --with_wandb True --wandb_project latency-sensitive-bench --wandb_group flappy-fs3-uniform_u1_3 --wandb_job_type sample_factory --wandb_tags factor_sweep flappy frame_stack uniform uniform_u1_3 fs3 seed13 --gym_id FlappyBird-v0 --env-fps 30 --obs-fps 30.0 --use_lidar False --normalize_obs True --audio_on False --obs_resize 84,84 --use-gpu-render True --simulator gpu --gpu-render-device auto --gpu-render-profile False --gpu-render-profile-interval 200 --pipe_gap 100 --bird_color yellow --pipe_color green --background day --frame_stack 3 --debug False --debug-timelimit-diagnostics False --hold-policy one_frame_then_noop --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_uniform_u1_3_fs3_seed13", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 13, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs3-uniform_u1_3", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "uniform", + "uniform_u1_3", + "fs3", + "seed13" + ], + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "frame_stack": 3, + "debug": false, + "debug_timelimit_diagnostics": false, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 13, + "add_latency_info": false, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_uniform_u1_3_fs3_seed13/episode_metrics.jsonl" + }, + "git_hash": "eb3a2e1efbd2aa03a60d7f44f5e18d8fdd0f5a2d", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/flappy/context_window/train/frame_stack/uniform_u1_3/fs3/seed_13", + "wandb_unique_id": "flappy-fs3-uniform_u1_3-s13" +} \ No newline at end of file diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs3:obs30:stride1:seed14/checkpoint_p0/checkpoint_000048840_25034752.pth b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs3:obs30:stride1:seed14/checkpoint_p0/checkpoint_000048840_25034752.pth new file mode 100644 index 0000000000000000000000000000000000000000..1d956461485733edfd6e13a3046049d4a4145c7c --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs3:obs30:stride1:seed14/checkpoint_p0/checkpoint_000048840_25034752.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f743d41fdc2f0ac7454272921984bd32454449fd998df01146b148d24c04f7c3 +size 7808061 diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs3:obs30:stride1:seed14/config.json b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs3:obs30:stride1:seed14/config.json new file mode 100644 index 0000000000000000000000000000000000000000..9355efb2e6ddce8226707b023457e242b5a8e2a7 --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs3:obs30:stride1:seed14/config.json @@ -0,0 +1,268 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_uniform_u1_3_fs3_seed14", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 14, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs3-uniform_u1_3", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "uniform", + "uniform_u1_3", + "fs3", + "seed14" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "screen_size": "", + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "score_limit": -1, + "frame_stack": 3, + "debug": false, + "debug_timelimit_diagnostics": false, + "max_episode_steps": 0, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 14, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": false, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_uniform_u1_3_fs3_seed14/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_flappy --experiment flappy_frame_stack_uniform_u1_3_fs3_seed14 --train_dir results/checkpoints_factor_sweeps/flappy/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-render-batch-size 128 --seed 14 --episode_metrics_path results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_uniform_u1_3_fs3_seed14/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 2 --num_envs_per_worker 1 --num_policies 1 --batch_size 4096 --rollout 128 --recurrence 1 --num_epochs 8 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 400 --learning_rate 0.00025 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.5 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 14 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --with_wandb True --wandb_project latency-sensitive-bench --wandb_group flappy-fs3-uniform_u1_3 --wandb_job_type sample_factory --wandb_tags factor_sweep flappy frame_stack uniform uniform_u1_3 fs3 seed14 --gym_id FlappyBird-v0 --env-fps 30 --obs-fps 30.0 --use_lidar False --normalize_obs True --audio_on False --obs_resize 84,84 --use-gpu-render True --simulator gpu --gpu-render-device auto --gpu-render-profile False --gpu-render-profile-interval 200 --pipe_gap 100 --bird_color yellow --pipe_color green --background day --frame_stack 3 --debug False --debug-timelimit-diagnostics False --hold-policy one_frame_then_noop --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_uniform_u1_3_fs3_seed14", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 14, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs3-uniform_u1_3", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "uniform", + "uniform_u1_3", + "fs3", + "seed14" + ], + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "frame_stack": 3, + "debug": false, + "debug_timelimit_diagnostics": false, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 14, + "add_latency_info": false, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_uniform_u1_3_fs3_seed14/episode_metrics.jsonl" + }, + "git_hash": "eb3a2e1efbd2aa03a60d7f44f5e18d8fdd0f5a2d", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/flappy/context_window/train/frame_stack/uniform_u1_3/fs3/seed_14", + "wandb_unique_id": "flappy-fs3-uniform_u1_3-s14" +} \ No newline at end of file diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs4:obs30:stride1:seed10/checkpoint_p0/checkpoint_000048848_25034752.pth b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs4:obs30:stride1:seed10/checkpoint_p0/checkpoint_000048848_25034752.pth new file mode 100644 index 0000000000000000000000000000000000000000..97635e9c1ea7e06a39d238648e682db9ce850b59 --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs4:obs30:stride1:seed10/checkpoint_p0/checkpoint_000048848_25034752.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eee721145bcbcf75b73353188bf079853c67cd09cf7bff61325783d17c1d3aeb +size 8171325 diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs4:obs30:stride1:seed10/config.json b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs4:obs30:stride1:seed10/config.json new file mode 100644 index 0000000000000000000000000000000000000000..9d5987026e9e94016f43e8f92b9c651407ec48c5 --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs4:obs30:stride1:seed10/config.json @@ -0,0 +1,268 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_uniform_u1_3_fs4_seed10", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 10, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs4-uniform_u1_3", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "uniform", + "uniform_u1_3", + "fs4", + "seed10" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "screen_size": "", + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "score_limit": -1, + "frame_stack": 4, + "debug": false, + "debug_timelimit_diagnostics": false, + "max_episode_steps": 0, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 10, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": false, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_uniform_u1_3_fs4_seed10/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_flappy --experiment flappy_frame_stack_uniform_u1_3_fs4_seed10 --train_dir results/checkpoints_factor_sweeps/flappy/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-render-batch-size 128 --seed 10 --episode_metrics_path results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_uniform_u1_3_fs4_seed10/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 2 --num_envs_per_worker 1 --num_policies 1 --batch_size 4096 --rollout 128 --recurrence 1 --num_epochs 8 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 400 --learning_rate 0.00025 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.5 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 10 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --with_wandb True --wandb_project latency-sensitive-bench --wandb_group flappy-fs4-uniform_u1_3 --wandb_job_type sample_factory --wandb_tags factor_sweep flappy frame_stack uniform uniform_u1_3 fs4 seed10 --gym_id FlappyBird-v0 --env-fps 30 --obs-fps 30.0 --use_lidar False --normalize_obs True --audio_on False --obs_resize 84,84 --use-gpu-render True --simulator gpu --gpu-render-device auto --gpu-render-profile False --gpu-render-profile-interval 200 --pipe_gap 100 --bird_color yellow --pipe_color green --background day --frame_stack 4 --debug False --debug-timelimit-diagnostics False --hold-policy one_frame_then_noop --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_uniform_u1_3_fs4_seed10", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 10, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs4-uniform_u1_3", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "uniform", + "uniform_u1_3", + "fs4", + "seed10" + ], + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "frame_stack": 4, + "debug": false, + "debug_timelimit_diagnostics": false, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 10, + "add_latency_info": false, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_uniform_u1_3_fs4_seed10/episode_metrics.jsonl" + }, + "git_hash": "284fe8ace24f0e8a40c03c5b559969abd7caeb29", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/flappy/context_window/train/frame_stack/uniform_u1_3/fs4/seed_10", + "wandb_unique_id": "flappy-fs4-uniform_u1_3-s10" +} \ No newline at end of file diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs4:obs30:stride1:seed11/checkpoint_p0/checkpoint_000048848_25034752.pth b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs4:obs30:stride1:seed11/checkpoint_p0/checkpoint_000048848_25034752.pth new file mode 100644 index 0000000000000000000000000000000000000000..8d5406ff44106a2c960dac0629f1b7bd282861bb --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs4:obs30:stride1:seed11/checkpoint_p0/checkpoint_000048848_25034752.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:de8c9bd021380f61bc0ec3e461f345ad0206defe9d8545e9f3640a3628ceec12 +size 8171325 diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs4:obs30:stride1:seed11/config.json b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs4:obs30:stride1:seed11/config.json new file mode 100644 index 0000000000000000000000000000000000000000..3eec71100a54a21d1b00bebd87826f5279b69a2b --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs4:obs30:stride1:seed11/config.json @@ -0,0 +1,268 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_uniform_u1_3_fs4_seed11", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 11, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs4-uniform_u1_3", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "uniform", + "uniform_u1_3", + "fs4", + "seed11" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "screen_size": "", + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "score_limit": -1, + "frame_stack": 4, + "debug": false, + "debug_timelimit_diagnostics": false, + "max_episode_steps": 0, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 11, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": false, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_uniform_u1_3_fs4_seed11/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_flappy --experiment flappy_frame_stack_uniform_u1_3_fs4_seed11 --train_dir results/checkpoints_factor_sweeps/flappy/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-render-batch-size 128 --seed 11 --episode_metrics_path results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_uniform_u1_3_fs4_seed11/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 2 --num_envs_per_worker 1 --num_policies 1 --batch_size 4096 --rollout 128 --recurrence 1 --num_epochs 8 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 400 --learning_rate 0.00025 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.5 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 11 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --with_wandb True --wandb_project latency-sensitive-bench --wandb_group flappy-fs4-uniform_u1_3 --wandb_job_type sample_factory --wandb_tags factor_sweep flappy frame_stack uniform uniform_u1_3 fs4 seed11 --gym_id FlappyBird-v0 --env-fps 30 --obs-fps 30.0 --use_lidar False --normalize_obs True --audio_on False --obs_resize 84,84 --use-gpu-render True --simulator gpu --gpu-render-device auto --gpu-render-profile False --gpu-render-profile-interval 200 --pipe_gap 100 --bird_color yellow --pipe_color green --background day --frame_stack 4 --debug False --debug-timelimit-diagnostics False --hold-policy one_frame_then_noop --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_uniform_u1_3_fs4_seed11", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 11, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs4-uniform_u1_3", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "uniform", + "uniform_u1_3", + "fs4", + "seed11" + ], + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "frame_stack": 4, + "debug": false, + "debug_timelimit_diagnostics": false, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 11, + "add_latency_info": false, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_uniform_u1_3_fs4_seed11/episode_metrics.jsonl" + }, + "git_hash": "284fe8ace24f0e8a40c03c5b559969abd7caeb29", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/flappy/context_window/train/frame_stack/uniform_u1_3/fs4/seed_11", + "wandb_unique_id": "flappy-fs4-uniform_u1_3-s11" +} \ No newline at end of file diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs4:obs30:stride1:seed12/checkpoint_p0/checkpoint_000048840_25034752.pth b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs4:obs30:stride1:seed12/checkpoint_p0/checkpoint_000048840_25034752.pth new file mode 100644 index 0000000000000000000000000000000000000000..edbd844226eb6931d2d11b34513b1b6d44dff7d7 --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs4:obs30:stride1:seed12/checkpoint_p0/checkpoint_000048840_25034752.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:72958ccd8ba7293d02a18ee2bc24dcd33813bbb598a930b7dcedb46e57486784 +size 8171325 diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs4:obs30:stride1:seed12/config.json b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs4:obs30:stride1:seed12/config.json new file mode 100644 index 0000000000000000000000000000000000000000..798c32220154cc0d51ad4d172513f7475c74edce --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs4:obs30:stride1:seed12/config.json @@ -0,0 +1,268 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_uniform_u1_3_fs4_seed12", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 12, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs4-uniform_u1_3", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "uniform", + "uniform_u1_3", + "fs4", + "seed12" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "screen_size": "", + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "score_limit": -1, + "frame_stack": 4, + "debug": false, + "debug_timelimit_diagnostics": false, + "max_episode_steps": 0, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 12, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": false, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_uniform_u1_3_fs4_seed12/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_flappy --experiment flappy_frame_stack_uniform_u1_3_fs4_seed12 --train_dir results/checkpoints_factor_sweeps/flappy/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-render-batch-size 128 --seed 12 --episode_metrics_path results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_uniform_u1_3_fs4_seed12/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 2 --num_envs_per_worker 1 --num_policies 1 --batch_size 4096 --rollout 128 --recurrence 1 --num_epochs 8 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 400 --learning_rate 0.00025 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.5 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 12 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --with_wandb True --wandb_project latency-sensitive-bench --wandb_group flappy-fs4-uniform_u1_3 --wandb_job_type sample_factory --wandb_tags factor_sweep flappy frame_stack uniform uniform_u1_3 fs4 seed12 --gym_id FlappyBird-v0 --env-fps 30 --obs-fps 30.0 --use_lidar False --normalize_obs True --audio_on False --obs_resize 84,84 --use-gpu-render True --simulator gpu --gpu-render-device auto --gpu-render-profile False --gpu-render-profile-interval 200 --pipe_gap 100 --bird_color yellow --pipe_color green --background day --frame_stack 4 --debug False --debug-timelimit-diagnostics False --hold-policy one_frame_then_noop --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_uniform_u1_3_fs4_seed12", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 12, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs4-uniform_u1_3", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "uniform", + "uniform_u1_3", + "fs4", + "seed12" + ], + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "frame_stack": 4, + "debug": false, + "debug_timelimit_diagnostics": false, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 12, + "add_latency_info": false, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_uniform_u1_3_fs4_seed12/episode_metrics.jsonl" + }, + "git_hash": "284fe8ace24f0e8a40c03c5b559969abd7caeb29", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/flappy/context_window/train/frame_stack/uniform_u1_3/fs4/seed_12", + "wandb_unique_id": "flappy-fs4-uniform_u1_3-s12" +} \ No newline at end of file diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs4:obs30:stride1:seed13/checkpoint_p0/checkpoint_000048832_25034752.pth b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs4:obs30:stride1:seed13/checkpoint_p0/checkpoint_000048832_25034752.pth new file mode 100644 index 0000000000000000000000000000000000000000..e6b94b9b8325931c5ca8de917456468268304d2c --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs4:obs30:stride1:seed13/checkpoint_p0/checkpoint_000048832_25034752.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:85eec7433a98e07a847163f68b1d48e5dd3aabc47257c66ec77876c8494534bb +size 8171325 diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs4:obs30:stride1:seed13/config.json b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs4:obs30:stride1:seed13/config.json new file mode 100644 index 0000000000000000000000000000000000000000..9ddb455bb1388e3fd77699d9877858d1aabce56e --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs4:obs30:stride1:seed13/config.json @@ -0,0 +1,268 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_uniform_u1_3_fs4_seed13", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 13, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs4-uniform_u1_3", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "uniform", + "uniform_u1_3", + "fs4", + "seed13" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "screen_size": "", + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "score_limit": -1, + "frame_stack": 4, + "debug": false, + "debug_timelimit_diagnostics": false, + "max_episode_steps": 0, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 13, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": false, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_uniform_u1_3_fs4_seed13/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_flappy --experiment flappy_frame_stack_uniform_u1_3_fs4_seed13 --train_dir results/checkpoints_factor_sweeps/flappy/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-render-batch-size 128 --seed 13 --episode_metrics_path results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_uniform_u1_3_fs4_seed13/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 2 --num_envs_per_worker 1 --num_policies 1 --batch_size 4096 --rollout 128 --recurrence 1 --num_epochs 8 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 400 --learning_rate 0.00025 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.5 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 13 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --with_wandb True --wandb_project latency-sensitive-bench --wandb_group flappy-fs4-uniform_u1_3 --wandb_job_type sample_factory --wandb_tags factor_sweep flappy frame_stack uniform uniform_u1_3 fs4 seed13 --gym_id FlappyBird-v0 --env-fps 30 --obs-fps 30.0 --use_lidar False --normalize_obs True --audio_on False --obs_resize 84,84 --use-gpu-render True --simulator gpu --gpu-render-device auto --gpu-render-profile False --gpu-render-profile-interval 200 --pipe_gap 100 --bird_color yellow --pipe_color green --background day --frame_stack 4 --debug False --debug-timelimit-diagnostics False --hold-policy one_frame_then_noop --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_uniform_u1_3_fs4_seed13", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 13, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs4-uniform_u1_3", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "uniform", + "uniform_u1_3", + "fs4", + "seed13" + ], + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "frame_stack": 4, + "debug": false, + "debug_timelimit_diagnostics": false, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 13, + "add_latency_info": false, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_uniform_u1_3_fs4_seed13/episode_metrics.jsonl" + }, + "git_hash": "284fe8ace24f0e8a40c03c5b559969abd7caeb29", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/flappy/context_window/train/frame_stack/uniform_u1_3/fs4/seed_13", + "wandb_unique_id": "flappy-fs4-uniform_u1_3-s13" +} \ No newline at end of file diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs4:obs30:stride1:seed14/checkpoint_p0/checkpoint_000048840_25034752.pth b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs4:obs30:stride1:seed14/checkpoint_p0/checkpoint_000048840_25034752.pth new file mode 100644 index 0000000000000000000000000000000000000000..27491c4f4b8e4fccb6633402c85e7ef0036eefa0 --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs4:obs30:stride1:seed14/checkpoint_p0/checkpoint_000048840_25034752.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dc77a82d55ab0ee9a1e4b258d50b0920a2e3234a07eab9a3b4eb91342b4f8986 +size 8171325 diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs4:obs30:stride1:seed14/config.json b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs4:obs30:stride1:seed14/config.json new file mode 100644 index 0000000000000000000000000000000000000000..764fefb23413429fc27f91e0187215fc0fea55e1 --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs4:obs30:stride1:seed14/config.json @@ -0,0 +1,268 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_uniform_u1_3_fs4_seed14", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 14, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs4-uniform_u1_3", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "uniform", + "uniform_u1_3", + "fs4", + "seed14" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "screen_size": "", + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "score_limit": -1, + "frame_stack": 4, + "debug": false, + "debug_timelimit_diagnostics": false, + "max_episode_steps": 0, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 14, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": false, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_uniform_u1_3_fs4_seed14/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_flappy --experiment flappy_frame_stack_uniform_u1_3_fs4_seed14 --train_dir results/checkpoints_factor_sweeps/flappy/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-render-batch-size 128 --seed 14 --episode_metrics_path results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_uniform_u1_3_fs4_seed14/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 2 --num_envs_per_worker 1 --num_policies 1 --batch_size 4096 --rollout 128 --recurrence 1 --num_epochs 8 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 400 --learning_rate 0.00025 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.5 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 14 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --with_wandb True --wandb_project latency-sensitive-bench --wandb_group flappy-fs4-uniform_u1_3 --wandb_job_type sample_factory --wandb_tags factor_sweep flappy frame_stack uniform uniform_u1_3 fs4 seed14 --gym_id FlappyBird-v0 --env-fps 30 --obs-fps 30.0 --use_lidar False --normalize_obs True --audio_on False --obs_resize 84,84 --use-gpu-render True --simulator gpu --gpu-render-device auto --gpu-render-profile False --gpu-render-profile-interval 200 --pipe_gap 100 --bird_color yellow --pipe_color green --background day --frame_stack 4 --debug False --debug-timelimit-diagnostics False --hold-policy one_frame_then_noop --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_uniform_u1_3_fs4_seed14", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 14, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs4-uniform_u1_3", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "uniform", + "uniform_u1_3", + "fs4", + "seed14" + ], + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "frame_stack": 4, + "debug": false, + "debug_timelimit_diagnostics": false, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 14, + "add_latency_info": false, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_uniform_u1_3_fs4_seed14/episode_metrics.jsonl" + }, + "git_hash": "284fe8ace24f0e8a40c03c5b559969abd7caeb29", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/flappy/context_window/train/frame_stack/uniform_u1_3/fs4/seed_14", + "wandb_unique_id": "flappy-fs4-uniform_u1_3-s14" +} \ No newline at end of file diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs5:obs30:stride1:seed10/checkpoint_p0/checkpoint_000048832_25034752.pth b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs5:obs30:stride1:seed10/checkpoint_p0/checkpoint_000048832_25034752.pth new file mode 100644 index 0000000000000000000000000000000000000000..7ae1ba40956558996ecbc5bb9c553702add88a81 --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs5:obs30:stride1:seed10/checkpoint_p0/checkpoint_000048832_25034752.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6539a0aad31f28c1c9baa57dc95fcb930c5d31cad1d029bc26865c37119b9073 +size 8534589 diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs5:obs30:stride1:seed10/config.json b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs5:obs30:stride1:seed10/config.json new file mode 100644 index 0000000000000000000000000000000000000000..0f71be201d04bbef0fb0568bdc11cae85349c76a --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs5:obs30:stride1:seed10/config.json @@ -0,0 +1,268 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_uniform_u1_3_fs5_seed10", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 10, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs5-uniform_u1_3", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "uniform", + "uniform_u1_3", + "fs5", + "seed10" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "screen_size": "", + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "score_limit": -1, + "frame_stack": 5, + "debug": false, + "debug_timelimit_diagnostics": false, + "max_episode_steps": 0, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 10, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": false, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_uniform_u1_3_fs5_seed10/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_flappy --experiment flappy_frame_stack_uniform_u1_3_fs5_seed10 --train_dir results/checkpoints_factor_sweeps/flappy/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-render-batch-size 128 --seed 10 --episode_metrics_path results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_uniform_u1_3_fs5_seed10/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 2 --num_envs_per_worker 1 --num_policies 1 --batch_size 4096 --rollout 128 --recurrence 1 --num_epochs 8 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 400 --learning_rate 0.00025 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.5 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 10 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --with_wandb True --wandb_project latency-sensitive-bench --wandb_group flappy-fs5-uniform_u1_3 --wandb_job_type sample_factory --wandb_tags factor_sweep flappy frame_stack uniform uniform_u1_3 fs5 seed10 --gym_id FlappyBird-v0 --env-fps 30 --obs-fps 30.0 --use_lidar False --normalize_obs True --audio_on False --obs_resize 84,84 --use-gpu-render True --simulator gpu --gpu-render-device auto --gpu-render-profile False --gpu-render-profile-interval 200 --pipe_gap 100 --bird_color yellow --pipe_color green --background day --frame_stack 5 --debug False --debug-timelimit-diagnostics False --hold-policy one_frame_then_noop --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_uniform_u1_3_fs5_seed10", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 10, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs5-uniform_u1_3", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "uniform", + "uniform_u1_3", + "fs5", + "seed10" + ], + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "frame_stack": 5, + "debug": false, + "debug_timelimit_diagnostics": false, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 10, + "add_latency_info": false, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_uniform_u1_3_fs5_seed10/episode_metrics.jsonl" + }, + "git_hash": "284fe8ace24f0e8a40c03c5b559969abd7caeb29", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/flappy/context_window/train/frame_stack/uniform_u1_3/fs5/seed_10", + "wandb_unique_id": "flappy-fs5-uniform_u1_3-s10" +} \ No newline at end of file diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs5:obs30:stride1:seed11/checkpoint_p0/checkpoint_000048760_25034752.pth b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs5:obs30:stride1:seed11/checkpoint_p0/checkpoint_000048760_25034752.pth new file mode 100644 index 0000000000000000000000000000000000000000..b71e3a42d37e2fa5ad19e93a4e7cca894736d1b1 --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs5:obs30:stride1:seed11/checkpoint_p0/checkpoint_000048760_25034752.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1dc744156f2e87ac5cb4aa5b2ee6c47303e7aa5d7dd1701243882e17e7a21682 +size 8534589 diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs5:obs30:stride1:seed11/config.json b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs5:obs30:stride1:seed11/config.json new file mode 100644 index 0000000000000000000000000000000000000000..082a9851c2dfd84338ed942bcce76f62e99521a6 --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs5:obs30:stride1:seed11/config.json @@ -0,0 +1,268 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_uniform_u1_3_fs5_seed11", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 11, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs5-uniform_u1_3", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "uniform", + "uniform_u1_3", + "fs5", + "seed11" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "screen_size": "", + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "score_limit": -1, + "frame_stack": 5, + "debug": false, + "debug_timelimit_diagnostics": false, + "max_episode_steps": 0, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 11, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": false, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_uniform_u1_3_fs5_seed11/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_flappy --experiment flappy_frame_stack_uniform_u1_3_fs5_seed11 --train_dir results/checkpoints_factor_sweeps/flappy/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-render-batch-size 128 --seed 11 --episode_metrics_path results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_uniform_u1_3_fs5_seed11/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 2 --num_envs_per_worker 1 --num_policies 1 --batch_size 4096 --rollout 128 --recurrence 1 --num_epochs 8 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 400 --learning_rate 0.00025 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.5 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 11 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --with_wandb True --wandb_project latency-sensitive-bench --wandb_group flappy-fs5-uniform_u1_3 --wandb_job_type sample_factory --wandb_tags factor_sweep flappy frame_stack uniform uniform_u1_3 fs5 seed11 --gym_id FlappyBird-v0 --env-fps 30 --obs-fps 30.0 --use_lidar False --normalize_obs True --audio_on False --obs_resize 84,84 --use-gpu-render True --simulator gpu --gpu-render-device auto --gpu-render-profile False --gpu-render-profile-interval 200 --pipe_gap 100 --bird_color yellow --pipe_color green --background day --frame_stack 5 --debug False --debug-timelimit-diagnostics False --hold-policy one_frame_then_noop --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_uniform_u1_3_fs5_seed11", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 11, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs5-uniform_u1_3", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "uniform", + "uniform_u1_3", + "fs5", + "seed11" + ], + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "frame_stack": 5, + "debug": false, + "debug_timelimit_diagnostics": false, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 11, + "add_latency_info": false, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_uniform_u1_3_fs5_seed11/episode_metrics.jsonl" + }, + "git_hash": "284fe8ace24f0e8a40c03c5b559969abd7caeb29", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/flappy/context_window/train/frame_stack/uniform_u1_3/fs5/seed_11", + "wandb_unique_id": "flappy-fs5-uniform_u1_3-s11" +} \ No newline at end of file diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs5:obs30:stride1:seed12/checkpoint_p0/checkpoint_000048832_25034752.pth b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs5:obs30:stride1:seed12/checkpoint_p0/checkpoint_000048832_25034752.pth new file mode 100644 index 0000000000000000000000000000000000000000..fba730afba11f1768229bf0271f53b74e486840a --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs5:obs30:stride1:seed12/checkpoint_p0/checkpoint_000048832_25034752.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3bdd8c5766c0e02ceed7ac0249c0ede55b855618245f08f0d11ec58617f101be +size 8534589 diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs5:obs30:stride1:seed12/config.json b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs5:obs30:stride1:seed12/config.json new file mode 100644 index 0000000000000000000000000000000000000000..e680277dd806704b2df686f54085d0e09fa9362f --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs5:obs30:stride1:seed12/config.json @@ -0,0 +1,268 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_uniform_u1_3_fs5_seed12", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 12, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs5-uniform_u1_3", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "uniform", + "uniform_u1_3", + "fs5", + "seed12" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "screen_size": "", + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "score_limit": -1, + "frame_stack": 5, + "debug": false, + "debug_timelimit_diagnostics": false, + "max_episode_steps": 0, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 12, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": false, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_uniform_u1_3_fs5_seed12/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_flappy --experiment flappy_frame_stack_uniform_u1_3_fs5_seed12 --train_dir results/checkpoints_factor_sweeps/flappy/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-render-batch-size 128 --seed 12 --episode_metrics_path results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_uniform_u1_3_fs5_seed12/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 2 --num_envs_per_worker 1 --num_policies 1 --batch_size 4096 --rollout 128 --recurrence 1 --num_epochs 8 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 400 --learning_rate 0.00025 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.5 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 12 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --with_wandb True --wandb_project latency-sensitive-bench --wandb_group flappy-fs5-uniform_u1_3 --wandb_job_type sample_factory --wandb_tags factor_sweep flappy frame_stack uniform uniform_u1_3 fs5 seed12 --gym_id FlappyBird-v0 --env-fps 30 --obs-fps 30.0 --use_lidar False --normalize_obs True --audio_on False --obs_resize 84,84 --use-gpu-render True --simulator gpu --gpu-render-device auto --gpu-render-profile False --gpu-render-profile-interval 200 --pipe_gap 100 --bird_color yellow --pipe_color green --background day --frame_stack 5 --debug False --debug-timelimit-diagnostics False --hold-policy one_frame_then_noop --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_uniform_u1_3_fs5_seed12", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 12, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs5-uniform_u1_3", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "uniform", + "uniform_u1_3", + "fs5", + "seed12" + ], + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "frame_stack": 5, + "debug": false, + "debug_timelimit_diagnostics": false, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 12, + "add_latency_info": false, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_uniform_u1_3_fs5_seed12/episode_metrics.jsonl" + }, + "git_hash": "284fe8ace24f0e8a40c03c5b559969abd7caeb29", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/flappy/context_window/train/frame_stack/uniform_u1_3/fs5/seed_12", + "wandb_unique_id": "flappy-fs5-uniform_u1_3-s12" +} \ No newline at end of file diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs5:obs30:stride1:seed13/checkpoint_p0/checkpoint_000048840_25034752.pth b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs5:obs30:stride1:seed13/checkpoint_p0/checkpoint_000048840_25034752.pth new file mode 100644 index 0000000000000000000000000000000000000000..4d701c7802b9c8417667accf593b36e145a7fdd0 --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs5:obs30:stride1:seed13/checkpoint_p0/checkpoint_000048840_25034752.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a7dcfd057b57b36f33ba8e136bd03ffe587d9682ade32c01d199eba130801cce +size 8534589 diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs5:obs30:stride1:seed13/config.json b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs5:obs30:stride1:seed13/config.json new file mode 100644 index 0000000000000000000000000000000000000000..e9e5f39edda25279a55d04738aef32c709616dca --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs5:obs30:stride1:seed13/config.json @@ -0,0 +1,268 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_uniform_u1_3_fs5_seed13", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 13, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs5-uniform_u1_3", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "uniform", + "uniform_u1_3", + "fs5", + "seed13" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "screen_size": "", + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "score_limit": -1, + "frame_stack": 5, + "debug": false, + "debug_timelimit_diagnostics": false, + "max_episode_steps": 0, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 13, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": false, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_uniform_u1_3_fs5_seed13/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_flappy --experiment flappy_frame_stack_uniform_u1_3_fs5_seed13 --train_dir results/checkpoints_factor_sweeps/flappy/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-render-batch-size 128 --seed 13 --episode_metrics_path results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_uniform_u1_3_fs5_seed13/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 2 --num_envs_per_worker 1 --num_policies 1 --batch_size 4096 --rollout 128 --recurrence 1 --num_epochs 8 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 400 --learning_rate 0.00025 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.5 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 13 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --with_wandb True --wandb_project latency-sensitive-bench --wandb_group flappy-fs5-uniform_u1_3 --wandb_job_type sample_factory --wandb_tags factor_sweep flappy frame_stack uniform uniform_u1_3 fs5 seed13 --gym_id FlappyBird-v0 --env-fps 30 --obs-fps 30.0 --use_lidar False --normalize_obs True --audio_on False --obs_resize 84,84 --use-gpu-render True --simulator gpu --gpu-render-device auto --gpu-render-profile False --gpu-render-profile-interval 200 --pipe_gap 100 --bird_color yellow --pipe_color green --background day --frame_stack 5 --debug False --debug-timelimit-diagnostics False --hold-policy one_frame_then_noop --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_uniform_u1_3_fs5_seed13", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 13, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs5-uniform_u1_3", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "uniform", + "uniform_u1_3", + "fs5", + "seed13" + ], + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "frame_stack": 5, + "debug": false, + "debug_timelimit_diagnostics": false, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 13, + "add_latency_info": false, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_uniform_u1_3_fs5_seed13/episode_metrics.jsonl" + }, + "git_hash": "284fe8ace24f0e8a40c03c5b559969abd7caeb29", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/flappy/context_window/train/frame_stack/uniform_u1_3/fs5/seed_13", + "wandb_unique_id": "flappy-fs5-uniform_u1_3-s13" +} \ No newline at end of file diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs5:obs30:stride1:seed14/checkpoint_p0/checkpoint_000048840_25034752.pth b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs5:obs30:stride1:seed14/checkpoint_p0/checkpoint_000048840_25034752.pth new file mode 100644 index 0000000000000000000000000000000000000000..fae5e7e2a3cda4e020f6cd44dd4b5de8ea57e7d6 --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs5:obs30:stride1:seed14/checkpoint_p0/checkpoint_000048840_25034752.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6253f0958661e6c93325e62980e0c0ce4f585c18dcd365ff078eddc908007cce +size 8534589 diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs5:obs30:stride1:seed14/config.json b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs5:obs30:stride1:seed14/config.json new file mode 100644 index 0000000000000000000000000000000000000000..cce1ac006e6220bb5108fbb7382237a7d31734fb --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs5:obs30:stride1:seed14/config.json @@ -0,0 +1,268 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_uniform_u1_3_fs5_seed14", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 14, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs5-uniform_u1_3", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "uniform", + "uniform_u1_3", + "fs5", + "seed14" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "screen_size": "", + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "score_limit": -1, + "frame_stack": 5, + "debug": false, + "debug_timelimit_diagnostics": false, + "max_episode_steps": 0, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 14, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": false, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_uniform_u1_3_fs5_seed14/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_flappy --experiment flappy_frame_stack_uniform_u1_3_fs5_seed14 --train_dir results/checkpoints_factor_sweeps/flappy/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-render-batch-size 128 --seed 14 --episode_metrics_path results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_uniform_u1_3_fs5_seed14/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 2 --num_envs_per_worker 1 --num_policies 1 --batch_size 4096 --rollout 128 --recurrence 1 --num_epochs 8 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 400 --learning_rate 0.00025 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.5 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 14 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --with_wandb True --wandb_project latency-sensitive-bench --wandb_group flappy-fs5-uniform_u1_3 --wandb_job_type sample_factory --wandb_tags factor_sweep flappy frame_stack uniform uniform_u1_3 fs5 seed14 --gym_id FlappyBird-v0 --env-fps 30 --obs-fps 30.0 --use_lidar False --normalize_obs True --audio_on False --obs_resize 84,84 --use-gpu-render True --simulator gpu --gpu-render-device auto --gpu-render-profile False --gpu-render-profile-interval 200 --pipe_gap 100 --bird_color yellow --pipe_color green --background day --frame_stack 5 --debug False --debug-timelimit-diagnostics False --hold-policy one_frame_then_noop --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_uniform_u1_3_fs5_seed14", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 14, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs5-uniform_u1_3", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "uniform", + "uniform_u1_3", + "fs5", + "seed14" + ], + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "frame_stack": 5, + "debug": false, + "debug_timelimit_diagnostics": false, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 14, + "add_latency_info": false, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_uniform_u1_3_fs5_seed14/episode_metrics.jsonl" + }, + "git_hash": "284fe8ace24f0e8a40c03c5b559969abd7caeb29", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/flappy/context_window/train/frame_stack/uniform_u1_3/fs5/seed_14", + "wandb_unique_id": "flappy-fs5-uniform_u1_3-s14" +} \ No newline at end of file diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs6:obs30:stride1:seed10/checkpoint_p0/checkpoint_000048840_25034752.pth b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs6:obs30:stride1:seed10/checkpoint_p0/checkpoint_000048840_25034752.pth new file mode 100644 index 0000000000000000000000000000000000000000..a8473b98a10cf22ee3ceafaf52aa3ed06a02574e --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs6:obs30:stride1:seed10/checkpoint_p0/checkpoint_000048840_25034752.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5c0948eead982c7c01ca8a5d72c87bd7a6a6d3ada29ecd12d20bec838461a1ea +size 8897853 diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs6:obs30:stride1:seed10/config.json b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs6:obs30:stride1:seed10/config.json new file mode 100644 index 0000000000000000000000000000000000000000..d4e7a9fc3110e3ba4cfc8fb9f28c1c7450718139 --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs6:obs30:stride1:seed10/config.json @@ -0,0 +1,268 @@ +{ + "help": false, + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_uniform_u1_3_fs6_seed10", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 10, + "num_policies": 1, + "async_rl": true, + "serial_mode": false, + "batched_sampling": true, + "num_batches_to_accumulate": 2, + "worker_num_splits": 1, + "policy_workers_per_policy": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "shuffle_minibatches": false, + "gamma": 0.99, + "reward_scale": 1.0, + "reward_clip": 1000.0, + "value_bootstrap": false, + "normalize_returns": true, + "exploration_loss_coeff": 0.003, + "value_loss_coeff": 0.5, + "kl_loss_coeff": 0.0, + "exploration_loss": "entropy", + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "with_vtrace": false, + "vtrace_rho": 1.0, + "vtrace_c": 1.0, + "optimizer": "adam", + "adam_eps": 1e-05, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "lr_schedule": "linear_decay", + "lr_schedule_kl_threshold": 0.008, + "lr_adaptive_min": 1e-06, + "lr_adaptive_max": 0.01, + "obs_subtract_mean": 0.0, + "obs_scale": 255.0, + "normalize_input": true, + "normalize_input_keys": null, + "decorrelate_experience_max_seconds": 0, + "decorrelate_envs_on_one_worker": true, + "actor_worker_gpus": [ + 0 + ], + "set_workers_cpu_affinity": true, + "force_envs_single_thread": false, + "default_niceness": 0, + "log_to_file": true, + "experiment_summaries_interval": 1, + "flush_summaries_interval": 30, + "stats_avg": 100, + "summaries_use_frameskip": true, + "heartbeat_interval": 20, + "heartbeat_reporting_interval": 180, + "train_for_env_steps": 25000000, + "train_for_seconds": 10000000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "load_checkpoint_kind": "latest", + "save_milestones_sec": -1, + "save_best_every_sec": 5, + "save_best_metric": "reward", + "save_best_after": 100000, + "benchmark": false, + "encoder_mlp_layers": [ + 512, + 512 + ], + "encoder_conv_architecture": "convnet_atari", + "encoder_conv_mlp_layers": [ + 512 + ], + "use_rnn": false, + "rnn_size": 512, + "rnn_type": "gru", + "rnn_num_layers": 1, + "decoder_mlp_layers": [], + "nonlinearity": "elu", + "policy_initialization": "orthogonal", + "policy_init_gain": 1.0, + "actor_critic_share_weights": true, + "adaptive_stddev": true, + "continuous_tanh_scale": 0.0, + "initial_stddev": 1.0, + "use_env_info_cache": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "env_frameskip": 1, + "env_framestack": 1, + "pixel_format": "CHW", + "use_record_episode_statistics": false, + "with_wandb": true, + "wandb_user": null, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs6-uniform_u1_3", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "uniform", + "uniform_u1_3", + "fs6", + "seed10" + ], + "with_pbt": false, + "pbt_mix_policies_in_one_env": true, + "pbt_period_env_steps": 5000000, + "pbt_start_mutation": 20000000, + "pbt_replace_fraction": 0.3, + "pbt_mutation_rate": 0.15, + "pbt_replace_reward_gap": 0.1, + "pbt_replace_reward_gap_absolute": 1e-06, + "pbt_optimize_gamma": false, + "pbt_target_objective": "true_objective", + "pbt_perturb_min": 1.1, + "pbt_perturb_max": 1.5, + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "screen_size": "", + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "score_limit": -1, + "frame_stack": 6, + "debug": false, + "debug_timelimit_diagnostics": false, + "max_episode_steps": 0, + "mode": "train", + "latency_type": "uniform_distribution", + "fixed_latency_ms": null, + "mean_latency_ms": null, + "std_latency_ms": null, + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 10, + "add_latency_info": false, + "max_pending_actions": null, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_latency_raw_frame_values": "0,1,2,3,4,5", + "eval_max_steps": 3600, + "eval_deterministic": true, + "eval_raw_reward": false, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_uniform_u1_3_fs6_seed10/episode_metrics.jsonl", + "command_line": "--mode train --algo APPO --env latency_flappy --experiment flappy_frame_stack_uniform_u1_3_fs6_seed10 --train_dir results/checkpoints_factor_sweeps/flappy/context_window --restart_behavior resume --device gpu --actor_worker_gpus 0 --env_gpu_observations True --env_gpu_actions True --gpu-render-batch-size 128 --seed 10 --episode_metrics_path results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_uniform_u1_3_fs6_seed10/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 2 --num_envs_per_worker 1 --num_policies 1 --batch_size 4096 --rollout 128 --recurrence 1 --num_epochs 8 --num_batches_per_epoch 8 --worker_num_splits 1 --max_policy_lag 400 --learning_rate 0.00025 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --value_loss_coeff 0.5 --max_grad_norm 0.5 --save_every_sec 600 --keep_checkpoints 5 --stats_avg 100 --experiment_summaries_interval 1 --batched_sampling True --async_rl True --use_rnn False --normalize_returns True --normalize_input True --latency-type uniform_distribution --min-latency-ms 33.333333333333336 --max-latency-ms 100.0 --latency-seed 10 --add-latency-info False --eval-episodes 100 --eval-parallel-envs 100 --eval-max-steps 3600 --eval-deterministic True --with_wandb True --wandb_project latency-sensitive-bench --wandb_group flappy-fs6-uniform_u1_3 --wandb_job_type sample_factory --wandb_tags factor_sweep flappy frame_stack uniform uniform_u1_3 fs6 seed10 --gym_id FlappyBird-v0 --env-fps 30 --obs-fps 30.0 --use_lidar False --normalize_obs True --audio_on False --obs_resize 84,84 --use-gpu-render True --simulator gpu --gpu-render-device auto --gpu-render-profile False --gpu-render-profile-interval 200 --pipe_gap 100 --bird_color yellow --pipe_color green --background day --frame_stack 6 --debug False --debug-timelimit-diagnostics False --hold-policy one_frame_then_noop --ordering-policy latest_ready", + "cli_args": { + "algo": "APPO", + "env": "latency_flappy", + "experiment": "flappy_frame_stack_uniform_u1_3_fs6_seed10", + "train_dir": "results/checkpoints_factor_sweeps/flappy/context_window", + "restart_behavior": "resume", + "device": "gpu", + "seed": 10, + "num_policies": 1, + "async_rl": true, + "batched_sampling": true, + "worker_num_splits": 1, + "max_policy_lag": 400, + "num_workers": 2, + "num_envs_per_worker": 1, + "batch_size": 4096, + "num_batches_per_epoch": 8, + "num_epochs": 8, + "rollout": 128, + "recurrence": 1, + "gamma": 0.99, + "normalize_returns": true, + "value_loss_coeff": 0.5, + "gae_lambda": 0.95, + "ppo_clip_ratio": 0.1, + "ppo_clip_value": 0.2, + "max_grad_norm": 0.5, + "learning_rate": 0.00025, + "normalize_input": true, + "actor_worker_gpus": [ + 0 + ], + "experiment_summaries_interval": 1, + "stats_avg": 100, + "train_for_env_steps": 25000000, + "save_every_sec": 600, + "keep_checkpoints": 5, + "use_rnn": false, + "env_gpu_actions": true, + "env_gpu_observations": true, + "with_wandb": true, + "wandb_project": "latency-sensitive-bench", + "wandb_group": "flappy-fs6-uniform_u1_3", + "wandb_job_type": "sample_factory", + "wandb_tags": [ + "factor_sweep", + "flappy", + "frame_stack", + "uniform", + "uniform_u1_3", + "fs6", + "seed10" + ], + "gym_id": "FlappyBird-v0", + "env_fps": 30.0, + "obs_fps": 30.0, + "use_lidar": false, + "normalize_obs": true, + "audio_on": false, + "obs_resize": "84,84", + "use_gpu_render": true, + "simulator": "gpu", + "gpu_render_device": "auto", + "gpu_render_batch_size": 128, + "gpu_render_profile": false, + "gpu_render_profile_interval": 200, + "pipe_gap": 100, + "bird_color": "yellow", + "pipe_color": "green", + "background": "day", + "frame_stack": 6, + "debug": false, + "debug_timelimit_diagnostics": false, + "mode": "train", + "latency_type": "uniform_distribution", + "min_latency_ms": 33.333333333333336, + "max_latency_ms": 100.0, + "latency_seed": 10, + "add_latency_info": false, + "hold_policy": "one_frame_then_noop", + "ordering_policy": "latest_ready", + "eval_episodes": 100, + "eval_parallel_envs": 100, + "eval_max_steps": 3600, + "eval_deterministic": true, + "episode_metrics_path": "results/checkpoints_factor_sweeps/flappy/context_window/flappy_frame_stack_uniform_u1_3_fs6_seed10/episode_metrics.jsonl" + }, + "git_hash": "284fe8ace24f0e8a40c03c5b559969abd7caeb29", + "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git", + "eval_env_frameskip": 1, + "output_dir": "outputs/factor_sweeps/flappy/context_window/train/frame_stack/uniform_u1_3/fs6/seed_10", + "wandb_unique_id": "flappy-fs6-uniform_u1_3-s10" +} \ No newline at end of file diff --git a/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs6:obs30:stride1:seed11/checkpoint_p0/checkpoint_000048840_25034752.pth b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs6:obs30:stride1:seed11/checkpoint_p0/checkpoint_000048840_25034752.pth new file mode 100644 index 0000000000000000000000000000000000000000..24f74e419934e92eedea8bf2144d291d8c649832 --- /dev/null +++ b/latency-aware/flappy/small-policy/factor_sweep:flappy:frame_stack:uniform_u1_3:fs6:obs30:stride1:seed11/checkpoint_p0/checkpoint_000048840_25034752.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:675ba25d1fdd826fe86f054b711a1c185c577f7a037e42838344bcc18e571b37 +size 8897853