dodox/dppo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

v0.5 to main (#10)

Browse Source 
* v0.5 (#9)

* update idql configs

* update awr configs

* update dipo configs

* update qsm configs

* update dqm configs

* update project version to 0.5.0
This commit is contained in:
Allen Z. Ren
2024-10-07 16:35:13 -04:00
committed by GitHub
parent dd14c5887c
commit e0842e71dc
267 changed files with 6769 additions and 1645 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cfg/gym/finetune/walker2d-v2/ft_awr_diffusion_mlp.yaml
+4 -4
View File
@@ -6,15 +6,15 @@ hydra:
_target_: agent.finetune.train_awr_diffusion_agent.TrainAWRDiffusionAgent
name: ${env_name}_awr_diffusion_mlp_ta${horizon_steps}_td${denoising_steps}
logdir: ${oc.env:DPPO_LOG_DIR}/gym-finetune/${name}/${now:%Y-%m-%d}_${now:%H-%M-%S}
logdir: ${oc.env:DPPO_LOG_DIR}/gym-finetune/${name}/${now:%Y-%m-%d}_${now:%H-%M-%S}_${seed}
base_policy_path: ${oc.env:DPPO_LOG_DIR}/gym-pretrain/walker2d-medium-v2_pre_diffusion_mlp_ta4_td20/2024-06-12_23-06-12/checkpoint/state_3000.pt
normalization_path: ${oc.env:DPPO_DATA_DIR}/gym/${env_name}/normalization.npz
seed: 42
device: cuda:0
env_name: walker2d-medium-v2
obs_dim: 17
action_dim: 6
transition_dim: ${action_dim}
denoising_steps: 20
cond_steps: 1
horizon_steps: 4
@@ -68,7 +68,7 @@ train:
max_adv_weight: 100
beta: 10
buffer_size: 5000
batch_size: 256
batch_size: 1000
replay_ratio: 64
critic_update_ratio: 4
@@ -82,7 +82,7 @@ model:
actor:
_target_: model.diffusion.mlp_diffusion.DiffusionMLP
horizon_steps: ${horizon_steps}
transition_dim: ${transition_dim}
action_dim: ${action_dim}
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
time_dim: 16
mlp_dims: [512, 512, 512]
cfg/gym/finetune/walker2d-v2/ft_dipo_diffusion_mlp.yaml
+7 -6
View File
@@ -6,15 +6,15 @@ hydra:
_target_: agent.finetune.train_dipo_diffusion_agent.TrainDIPODiffusionAgent
name: ${env_name}_dipo_diffusion_mlp_ta${horizon_steps}_td${denoising_steps}
logdir: ${oc.env:DPPO_LOG_DIR}/gym-finetune/${name}/${now:%Y-%m-%d}_${now:%H-%M-%S}
logdir: ${oc.env:DPPO_LOG_DIR}/gym-finetune/${name}/${now:%Y-%m-%d}_${now:%H-%M-%S}_${seed}
base_policy_path: ${oc.env:DPPO_LOG_DIR}/gym-pretrain/walker2d-medium-v2_pre_diffusion_mlp_ta4_td20/2024-06-12_23-06-12/checkpoint/state_3000.pt
normalization_path: ${oc.env:DPPO_DATA_DIR}/gym/${env_name}/normalization.npz
seed: 42
device: cuda:0
env_name: walker2d-medium-v2
obs_dim: 17
action_dim: 6
transition_dim: ${action_dim}
denoising_steps: 20
cond_steps: 1
horizon_steps: 4
@@ -65,11 +65,12 @@ train:
num: 0
# DIPO specific
scale_reward_factor: 0.01
eta: 0.0001
target_ema_rate: 0.005
buffer_size: 1000000
action_lr: 0.0001
action_gradient_steps: 10
buffer_size: 400000
batch_size: 5000
replay_ratio: 64
batch_size: 1000
model:
_target_: model.diffusion.diffusion_dipo.DIPODiffusion
@@ -81,7 +82,7 @@ model:
actor:
_target_: model.diffusion.mlp_diffusion.DiffusionMLP
horizon_steps: ${horizon_steps}
transition_dim: ${transition_dim}
action_dim: ${action_dim}
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
time_dim: 16
mlp_dims: [512, 512, 512]
cfg/gym/finetune/walker2d-v2/ft_dql_diffusion_mlp.yaml
+7 -6
View File
@@ -6,15 +6,15 @@ hydra:
_target_: agent.finetune.train_dql_diffusion_agent.TrainDQLDiffusionAgent
name: ${env_name}_dql_diffusion_mlp_ta${horizon_steps}_td${denoising_steps}
logdir: ${oc.env:DPPO_LOG_DIR}/gym-finetune/${name}/${now:%Y-%m-%d}_${now:%H-%M-%S}
logdir: ${oc.env:DPPO_LOG_DIR}/gym-finetune/${name}/${now:%Y-%m-%d}_${now:%H-%M-%S}_${seed}
base_policy_path: ${oc.env:DPPO_LOG_DIR}/gym-pretrain/walker2d-medium-v2_pre_diffusion_mlp_ta4_td20/2024-06-12_23-06-12/checkpoint/state_3000.pt
normalization_path: ${oc.env:DPPO_DATA_DIR}/gym/${env_name}/normalization.npz
seed: 42
device: cuda:0
env_name: walker2d-medium-v2
obs_dim: 17
action_dim: 6
transition_dim: ${action_dim}
denoising_steps: 20
cond_steps: 1
horizon_steps: 4
@@ -65,10 +65,11 @@ train:
num: 0
# DQL specific
scale_reward_factor: 0.01
target_ema_rate: 0.005
buffer_size: 1000000
eta: 1.0
buffer_size: 400000
batch_size: 5000
replay_ratio: 64
replay_ratio: 16
batch_size: 1000
model:
_target_: model.diffusion.diffusion_dql.DQLDiffusion
@@ -80,7 +81,7 @@ model:
actor:
_target_: model.diffusion.mlp_diffusion.DiffusionMLP
horizon_steps: ${horizon_steps}
transition_dim: ${transition_dim}
action_dim: ${action_dim}
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
time_dim: 16
mlp_dims: [512, 512, 512]
cfg/gym/finetune/walker2d-v2/ft_idql_diffusion_mlp.yaml
+6 -6
View File
@@ -6,15 +6,15 @@ hydra:
_target_: agent.finetune.train_idql_diffusion_agent.TrainIDQLDiffusionAgent
name: ${env_name}_idql_diffusion_mlp_ta${horizon_steps}_td${denoising_steps}
logdir: ${oc.env:DPPO_LOG_DIR}/gym-finetune/${name}/${now:%Y-%m-%d}_${now:%H-%M-%S}
logdir: ${oc.env:DPPO_LOG_DIR}/gym-finetune/${name}/${now:%Y-%m-%d}_${now:%H-%M-%S}_${seed}
base_policy_path: ${oc.env:DPPO_LOG_DIR}/gym-pretrain/walker2d-medium-v2_pre_diffusion_mlp_ta4_td20/2024-06-12_23-06-12/checkpoint/state_3000.pt
normalization_path: ${oc.env:DPPO_DATA_DIR}/gym/${env_name}/normalization.npz
seed: 42
device: cuda:0
env_name: walker2d-medium-v2
obs_dim: 17
action_dim: 6
transition_dim: ${action_dim}
denoising_steps: 20
cond_steps: 1
horizon_steps: 4
@@ -69,9 +69,9 @@ train:
eval_sample_num: 20 # how many samples to score during eval
critic_tau: 0.001 # rate of target q network update
use_expectile_exploration: True
buffer_size: 5000
batch_size: 512
replay_ratio: 16
buffer_size: 25000 # * n_envs
replay_ratio: 128
batch_size: 1000
model:
_target_: model.diffusion.diffusion_idql.IDQLDiffusion
@@ -83,7 +83,7 @@ model:
actor:
_target_: model.diffusion.mlp_diffusion.DiffusionMLP
horizon_steps: ${horizon_steps}
transition_dim: ${transition_dim}
action_dim: ${action_dim}
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
time_dim: 16
mlp_dims: [512, 512, 512]
cfg/gym/finetune/walker2d-v2/ft_ppo_diffusion_mlp.yaml
+3 -3
View File
@@ -6,15 +6,15 @@ hydra:
_target_: agent.finetune.train_ppo_diffusion_agent.TrainPPODiffusionAgent
name: ${env_name}_ppo_diffusion_mlp_ta${horizon_steps}_td${denoising_steps}_tdf${ft_denoising_steps}
logdir: ${oc.env:DPPO_LOG_DIR}/gym-finetune/${name}/${now:%Y-%m-%d}_${now:%H-%M-%S}
logdir: ${oc.env:DPPO_LOG_DIR}/gym-finetune/${name}/${now:%Y-%m-%d}_${now:%H-%M-%S}_${seed}
base_policy_path: ${oc.env:DPPO_LOG_DIR}/gym-pretrain/walker2d-medium-v2_pre_diffusion_mlp_ta4_td20/2024-06-12_23-06-12/checkpoint/state_3000.pt
normalization_path: ${oc.env:DPPO_DATA_DIR}/gym/${env_name}/normalization.npz
seed: 42
device: cuda:0
env_name: walker2d-medium-v2
obs_dim: 17
action_dim: 6
transition_dim: ${action_dim}
denoising_steps: 20
ft_denoising_steps: 10
cond_steps: 1
@@ -93,7 +93,7 @@ model:
residual_style: True
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
horizon_steps: ${horizon_steps}
transition_dim: ${transition_dim}
action_dim: ${action_dim}
critic:
_target_: model.common.critic.CriticObs
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
cfg/gym/finetune/walker2d-v2/ft_qsm_diffusion_mlp.yaml
+8 -8
View File
@@ -6,15 +6,15 @@ hydra:
_target_: agent.finetune.train_qsm_diffusion_agent.TrainQSMDiffusionAgent
name: ${env_name}_qsm_diffusion_mlp_ta${horizon_steps}_td${denoising_steps}
logdir: ${oc.env:DPPO_LOG_DIR}/gym-finetune/${name}/${now:%Y-%m-%d}_${now:%H-%M-%S}
logdir: ${oc.env:DPPO_LOG_DIR}/gym-finetune/${name}/${now:%Y-%m-%d}_${now:%H-%M-%S}_${seed}
base_policy_path: ${oc.env:DPPO_LOG_DIR}/gym-pretrain/walker2d-medium-v2_pre_diffusion_mlp_ta4_td20/2024-06-12_23-06-12/checkpoint/state_3000.pt
normalization_path: ${oc.env:DPPO_DATA_DIR}/gym/${env_name}/normalization.npz
seed: 42
device: cuda:0
env_name: walker2d-medium-v2
obs_dim: 17
action_dim: 6
transition_dim: ${action_dim}
denoising_steps: 20
cond_steps: 1
horizon_steps: 4
@@ -65,11 +65,11 @@ train:
num: 0
# QSM specific
scale_reward_factor: 0.01
q_grad_coeff: 50
critic_tau: 0.005 # rate of target q network update
buffer_size: 5000
batch_size: 256
replay_ratio: 32
q_grad_coeff: 10
critic_tau: 0.005
buffer_size: 25000
replay_ratio: 16
batch_size: 1000
model:
_target_: model.diffusion.diffusion_qsm.QSMDiffusion
@@ -81,7 +81,7 @@ model:
actor:
_target_: model.diffusion.mlp_diffusion.DiffusionMLP
horizon_steps: ${horizon_steps}
transition_dim: ${transition_dim}
action_dim: ${action_dim}
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
time_dim: 16
mlp_dims: [512, 512, 512]
cfg/gym/finetune/walker2d-v2/ft_rlpd_mlp.yaml
+103
View File
@@ -0,0 +1,103 @@
defaults:
- _self_
hydra:
run:
dir: ${logdir}
_target_: agent.finetune.train_rlpd_agent.TrainRLPDAgent
name: ${env_name}_rlpd_mlp_ta${horizon_steps}_td${denoising_steps}
logdir: ${oc.env:DPPO_LOG_DIR}/gym-finetune/${name}/${now:%Y-%m-%d}_${now:%H-%M-%S}_${seed}
normalization_path: ${oc.env:DPPO_DATA_DIR}/gym/${env_name}/normalization.npz
offline_dataset_path: ${oc.env:DPPO_DATA_DIR}/gym/${env_name}/train.npz
seed: 42
device: cuda:0
env_name: walker2d-medium-v2
obs_dim: 17
action_dim: 6
denoising_steps: 20
cond_steps: 1
horizon_steps: 1
act_steps: 1
env:
n_envs: 40
name: ${env_name}
max_episode_steps: 1000
reset_at_iteration: False
save_video: False
best_reward_threshold_for_success: 3
wrappers:
mujoco_locomotion_lowdim:
normalization_path: ${normalization_path}
multi_step:
n_obs_steps: ${cond_steps}
n_action_steps: ${act_steps}
max_episode_steps: ${env.max_episode_steps}
reset_within_step: True
wandb:
entity: ${oc.env:DPPO_WANDB_ENTITY}
project: rlpd-gym-${env_name}-finetune
run: ${now:%H-%M-%S}_${name}
train:
n_train_itr: 1000
n_critic_warmup_itr: 5
n_steps: 2000
gamma: 0.99
actor_lr: 1e-4
actor_weight_decay: 0
actor_lr_scheduler:
first_cycle_steps: 1000
warmup_steps: 10
min_lr: 1e-4
critic_lr: 1e-3
critic_weight_decay: 0
critic_lr_scheduler:
first_cycle_steps: 1000
warmup_steps: 10
min_lr: 1e-3
save_model_freq: 100
val_freq: 10
render:
freq: 1
num: 0
# RLPD specific
batch_size: 512
entropy_temperature: 1.0 # alpha in RLPD paper
target_ema_rate: 0.005 # rho in RLPD paper
scale_reward_factor: 1.0 # multiply reward by this amount for more stable value estimation
replay_ratio: 64 # number of batches to sample for each learning update
buffer_size: 1000000
model:
_target_: model.rl.gaussian_rlpd.RLPD_Gaussian
randn_clip_value: 3
actor:
_target_: model.common.mlp_gaussian.Gaussian_MLP
mlp_dims: [512, 512, 512]
activation_type: ReLU
residual_style: True
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
horizon_steps: ${horizon_steps}
action_dim: ${action_dim}
critic:
_target_: model.common.critic.CriticObsAct
action_dim: ${action_dim}
action_steps: ${act_steps}
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
mlp_dims: [256, 256, 256]
activation_type: Mish
residual_style: True
use_layernorm: True
horizon_steps: ${horizon_steps}
device: ${device}
n_critics: 2 # Ensemble size for critic models
offline_dataset:
_target_: agent.dataset.sequence.StitchedSequenceQLearningDataset
dataset_path: ${offline_dataset_path}
horizon_steps: ${horizon_steps}
cond_steps: ${cond_steps}
device: ${device}
cfg/gym/finetune/walker2d-v2/ft_rwr_diffusion_mlp.yaml
+3 -3
View File
@@ -6,15 +6,15 @@ hydra:
_target_: agent.finetune.train_rwr_diffusion_agent.TrainRWRDiffusionAgent
name: ${env_name}_rwr_diffusion_mlp_ta${horizon_steps}_td${denoising_steps}
logdir: ${oc.env:DPPO_LOG_DIR}/gym-finetune/${name}/${now:%Y-%m-%d}_${now:%H-%M-%S}
logdir: ${oc.env:DPPO_LOG_DIR}/gym-finetune/${name}/${now:%Y-%m-%d}_${now:%H-%M-%S}_${seed}
base_policy_path: ${oc.env:DPPO_LOG_DIR}/gym-pretrain/walker2d-medium-v2_pre_diffusion_mlp_ta4_td20/2024-06-12_23-06-12/checkpoint/state_3000.pt
normalization_path: ${oc.env:DPPO_DATA_DIR}/gym/${env_name}/normalization.npz
seed: 42
device: cuda:0
env_name: walker2d-medium-v2
obs_dim: 17
action_dim: 6
transition_dim: ${action_dim}
denoising_steps: 20
cond_steps: 1
horizon_steps: 4
@@ -73,7 +73,7 @@ model:
network:
_target_: model.diffusion.mlp_diffusion.DiffusionMLP
horizon_steps: ${horizon_steps}
transition_dim: ${transition_dim}
action_dim: ${action_dim}
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
time_dim: 16
mlp_dims: [512, 512, 512]
cfg/gym/finetune/walker2d-v2/ppo_diffusion_mlp.yaml
+3 -3
View File
@@ -6,14 +6,14 @@ hydra:
_target_: agent.finetune.train_ppo_diffusion_agent.TrainPPODiffusionAgent
name: ${env_name}_nopre_ppo_diffusion_mlp_ta${horizon_steps}_td${denoising_steps}
logdir: ${oc.env:DPPO_LOG_DIR}/gym-finetune/${name}/${now:%Y-%m-%d}_${now:%H-%M-%S}
logdir: ${oc.env:DPPO_LOG_DIR}/gym-finetune/${name}/${now:%Y-%m-%d}_${now:%H-%M-%S}_${seed}
normalization_path: ${oc.env:DPPO_DATA_DIR}/gym/${env_name}/normalization.npz
seed: 42
device: cuda:0
env_name: walker2d-medium-v2
obs_dim: 17
action_dim: 6
transition_dim: ${action_dim}
denoising_steps: 20
ft_denoising_steps: 20
cond_steps: 1
@@ -86,7 +86,7 @@ model:
actor:
_target_: model.diffusion.mlp_diffusion.DiffusionMLP
horizon_steps: ${horizon_steps}
transition_dim: ${transition_dim}
action_dim: ${action_dim}
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
time_dim: 16
mlp_dims: [512, 512, 512]
cfg/gym/finetune/walker2d-v2/ppo_gaussian_mlp.yaml
+4 -4
View File
@@ -6,14 +6,14 @@ hydra:
_target_: agent.finetune.train_ppo_gaussian_agent.TrainPPOGaussianAgent
name: ${env_name}_nopre_ppo_gaussian_mlp_ta${horizon_steps}
logdir: ${oc.env:DPPO_LOG_DIR}/gym-finetune/${name}/${now:%Y-%m-%d}_${now:%H-%M-%S}
logdir: ${oc.env:DPPO_LOG_DIR}/gym-finetune/${name}/${now:%Y-%m-%d}_${now:%H-%M-%S}_${seed}
normalization_path: ${oc.env:DPPO_DATA_DIR}/gym/${env_name}/normalization.npz
seed: 42
device: cuda:0
env_name: walker2d-medium-v2
obs_dim: 17
action_dim: 6
transition_dim: ${action_dim}
cond_steps: 1
horizon_steps: 1
act_steps: 1
@@ -79,10 +79,10 @@ model:
_target_: model.common.mlp_gaussian.Gaussian_MLP
mlp_dims: [512, 512, 512]
activation_type: ReLU
residual_style: True
residual_style: False # with new logvar head
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
horizon_steps: ${horizon_steps}
transition_dim: ${transition_dim}
action_dim: ${action_dim}
critic:
_target_: model.common.critic.CriticObs
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}