dodox/dppo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

v0.5 to main (#10)

Browse Source 
* v0.5 (#9)

* update idql configs

* update awr configs

* update dipo configs

* update qsm configs

* update dqm configs

* update project version to 0.5.0
This commit is contained in:
Allen Z. Ren
2024-10-07 16:35:13 -04:00
committed by GitHub
parent dd14c5887c
commit e0842e71dc
267 changed files with 6769 additions and 1645 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cfg/gym/finetune/halfcheetah-v2/calql_mlp_online.yaml
+117
View File
@@ -0,0 +1,117 @@
defaults:
- _self_
hydra:
run:
dir: ${logdir}
_target_: agent.finetune.train_calql_agent.TrainCalQLAgent
name: ${env_name}_calql_mlp_ta${horizon_steps}
logdir: ${oc.env:DPPO_LOG_DIR}/gym-finetune/${name}/${now:%Y-%m-%d}_${now:%H-%M-%S}_${seed}
base_policy_path:
normalization_path: ${oc.env:DPPO_DATA_DIR}/gym/${env_name}/normalization.npz
offline_dataset_path: ${oc.env:DPPO_DATA_DIR}/gym/${env_name}/train.npz
seed: 42
device: cuda:0
env_name: halfcheetah-medium-v2
obs_dim: 17
action_dim: 6
cond_steps: 1
horizon_steps: 1
act_steps: 1
env:
n_envs: 1
name: ${env_name}
max_episode_steps: 1000
reset_at_iteration: False
save_video: False
best_reward_threshold_for_success: 3
wrappers:
mujoco_locomotion_lowdim:
normalization_path: ${normalization_path}
multi_step:
n_obs_steps: ${cond_steps}
n_action_steps: ${act_steps}
max_episode_steps: ${env.max_episode_steps}
reset_within_step: True
wandb:
entity: ${oc.env:DPPO_WANDB_ENTITY}
project: calql-${env_name}
run: ${now:%H-%M-%S}_${name}
train:
n_train_itr: 10000
n_steps: 1 # not used
n_episode_per_epoch: 1
gamma: 0.99
actor_lr: 1e-4
actor_weight_decay: 0
actor_lr_scheduler:
first_cycle_steps: 1000
warmup_steps: 10
min_lr: 1e-4
critic_lr: 3e-4
critic_weight_decay: 0
critic_lr_scheduler:
first_cycle_steps: 1000
warmup_steps: 10
min_lr: 3e-4
save_model_freq: 100
val_freq: 10
render:
freq: 1
num: 0
log_freq: 1
# CalQL specific
train_online: True
batch_size: 256
n_random_actions: 4
target_ema_rate: 0.005
scale_reward_factor: 1.0
num_update: 1000
buffer_size: 1000000
online_utd_ratio: 1
n_eval_episode: 10
n_explore_steps: 0
target_entropy: ${eval:'- ${action_dim} * ${act_steps}'}
init_temperature: 1
automatic_entropy_tuning: True
model:
_target_: model.rl.gaussian_calql.CalQL_Gaussian
randn_clip_value: 3
cql_min_q_weight: 5.0
tanh_output: True
network_path: ${base_policy_path}
actor:
_target_: model.common.mlp_gaussian.Gaussian_MLP
mlp_dims: [256, 256]
activation_type: ReLU
tanh_output: False # squash after sampling instead
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
horizon_steps: ${horizon_steps}
std_max: 7.3891
std_min: 0.0067
critic:
_target_: model.common.critic.CriticObsAct
mlp_dims: [256, 256]
activation_type: ReLU
use_layernorm: True
double_q: True
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
action_dim: ${action_dim}
action_steps: ${act_steps}
horizon_steps: ${horizon_steps}
device: ${device}
offline_dataset:
_target_: agent.dataset.sequence.StitchedSequenceQLearningDataset
dataset_path: ${offline_dataset_path}
horizon_steps: ${horizon_steps}
cond_steps: ${cond_steps}
device: ${device}
discount_factor: ${train.gamma}
get_mc_return: True
cfg/gym/finetune/halfcheetah-v2/ft_awr_diffusion_mlp.yaml
+4 -4
View File
@@ -6,15 +6,15 @@ hydra:
_target_: agent.finetune.train_awr_diffusion_agent.TrainAWRDiffusionAgent
name: ${env_name}_awr_diffusion_mlp_ta${horizon_steps}_td${denoising_steps}
logdir: ${oc.env:DPPO_LOG_DIR}/gym-finetune/${name}/${now:%Y-%m-%d}_${now:%H-%M-%S}
logdir: ${oc.env:DPPO_LOG_DIR}/gym-finetune/${name}/${now:%Y-%m-%d}_${now:%H-%M-%S}_${seed}
base_policy_path: ${oc.env:DPPO_LOG_DIR}/gym-pretrain/halfcheetah-medium-v2_pre_diffusion_mlp_ta4_td20/2024-06-12_23-04-42/checkpoint/state_3000.pt
normalization_path: ${oc.env:DPPO_DATA_DIR}/gym/${env_name}/normalization.npz
seed: 42
device: cuda:0
env_name: halfcheetah-medium-v2
obs_dim: 17
action_dim: 6
transition_dim: ${action_dim}
denoising_steps: 20
cond_steps: 1
horizon_steps: 4
@@ -68,7 +68,7 @@ train:
max_adv_weight: 100
beta: 10
buffer_size: 5000
batch_size: 256
batch_size: 1000
replay_ratio: 64
critic_update_ratio: 4
@@ -82,7 +82,7 @@ model:
actor:
_target_: model.diffusion.mlp_diffusion.DiffusionMLP
horizon_steps: ${horizon_steps}
transition_dim: ${transition_dim}
action_dim: ${action_dim}
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
time_dim: 16
mlp_dims: [512, 512, 512]
cfg/gym/finetune/halfcheetah-v2/ft_dipo_diffusion_mlp.yaml
+7 -6
View File
@@ -6,15 +6,15 @@ hydra:
_target_: agent.finetune.train_dipo_diffusion_agent.TrainDIPODiffusionAgent
name: ${env_name}_dipo_diffusion_mlp_ta${horizon_steps}_td${denoising_steps}
logdir: ${oc.env:DPPO_LOG_DIR}/gym-finetune/${name}/${now:%Y-%m-%d}_${now:%H-%M-%S}
logdir: ${oc.env:DPPO_LOG_DIR}/gym-finetune/${name}/${now:%Y-%m-%d}_${now:%H-%M-%S}_${seed}
base_policy_path: ${oc.env:DPPO_LOG_DIR}/gym-pretrain/halfcheetah-medium-v2_pre_diffusion_mlp_ta4_td20/2024-06-12_23-04-42/checkpoint/state_3000.pt
normalization_path: ${oc.env:DPPO_DATA_DIR}/gym/${env_name}/normalization.npz
seed: 42
device: cuda:0
env_name: halfcheetah-medium-v2
obs_dim: 17
action_dim: 6
transition_dim: ${action_dim}
denoising_steps: 20
cond_steps: 1
horizon_steps: 4
@@ -65,11 +65,12 @@ train:
num: 0
# DIPO specific
scale_reward_factor: 0.01
eta: 0.0001
target_ema_rate: 0.005
buffer_size: 1000000
action_lr: 0.0001
action_gradient_steps: 10
buffer_size: 400000
batch_size: 5000
replay_ratio: 64
batch_size: 1000
model:
_target_: model.diffusion.diffusion_dipo.DIPODiffusion
@@ -81,7 +82,7 @@ model:
actor:
_target_: model.diffusion.mlp_diffusion.DiffusionMLP
horizon_steps: ${horizon_steps}
transition_dim: ${transition_dim}
action_dim: ${action_dim}
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
time_dim: 16
mlp_dims: [512, 512, 512]
cfg/gym/finetune/halfcheetah-v2/ft_dql_diffusion_mlp.yaml
+7 -6
View File
@@ -6,15 +6,15 @@ hydra:
_target_: agent.finetune.train_dql_diffusion_agent.TrainDQLDiffusionAgent
name: ${env_name}_dql_diffusion_mlp_ta${horizon_steps}_td${denoising_steps}
logdir: ${oc.env:DPPO_LOG_DIR}/gym-finetune/${name}/${now:%Y-%m-%d}_${now:%H-%M-%S}
logdir: ${oc.env:DPPO_LOG_DIR}/gym-finetune/${name}/${now:%Y-%m-%d}_${now:%H-%M-%S}_${seed}
base_policy_path: ${oc.env:DPPO_LOG_DIR}/gym-pretrain/halfcheetah-medium-v2_pre_diffusion_mlp_ta4_td20/2024-06-12_23-04-42/checkpoint/state_3000.pt
normalization_path: ${oc.env:DPPO_DATA_DIR}/gym/${env_name}/normalization.npz
seed: 42
device: cuda:0
env_name: halfcheetah-medium-v2
obs_dim: 17
action_dim: 6
transition_dim: ${action_dim}
denoising_steps: 20
cond_steps: 1
horizon_steps: 4
@@ -65,10 +65,11 @@ train:
num: 0
# DQL specific
scale_reward_factor: 0.01
target_ema_rate: 0.005
buffer_size: 1000000
eta: 1.0
buffer_size: 400000
batch_size: 5000
replay_ratio: 64
replay_ratio: 16
batch_size: 1000
model:
_target_: model.diffusion.diffusion_dql.DQLDiffusion
@@ -80,7 +81,7 @@ model:
actor:
_target_: model.diffusion.mlp_diffusion.DiffusionMLP
horizon_steps: ${horizon_steps}
transition_dim: ${transition_dim}
action_dim: ${action_dim}
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
time_dim: 16
mlp_dims: [512, 512, 512]
cfg/gym/finetune/halfcheetah-v2/ft_idql_diffusion_mlp.yaml
+6 -6
View File
@@ -6,15 +6,15 @@ hydra:
_target_: agent.finetune.train_idql_diffusion_agent.TrainIDQLDiffusionAgent
name: ${env_name}_idql_diffusion_mlp_ta${horizon_steps}_td${denoising_steps}
logdir: ${oc.env:DPPO_LOG_DIR}/gym-finetune/${name}/${now:%Y-%m-%d}_${now:%H-%M-%S}
logdir: ${oc.env:DPPO_LOG_DIR}/gym-finetune/${name}/${now:%Y-%m-%d}_${now:%H-%M-%S}_${seed}
base_policy_path: ${oc.env:DPPO_LOG_DIR}/gym-pretrain/halfcheetah-medium-v2_pre_diffusion_mlp_ta4_td20/2024-06-12_23-04-42/checkpoint/state_3000.pt
normalization_path: ${oc.env:DPPO_DATA_DIR}/gym/${env_name}/normalization.npz
seed: 42
device: cuda:0
env_name: halfcheetah-medium-v2
obs_dim: 17
action_dim: 6
transition_dim: ${action_dim}
denoising_steps: 20
cond_steps: 1
horizon_steps: 4
@@ -69,9 +69,9 @@ train:
eval_sample_num: 20 # how many samples to score during eval
critic_tau: 0.001 # rate of target q network update
use_expectile_exploration: True
buffer_size: 5000
batch_size: 512
replay_ratio: 16
buffer_size: 25000 # * n_envs
replay_ratio: 128
batch_size: 1000
model:
_target_: model.diffusion.diffusion_idql.IDQLDiffusion
@@ -83,7 +83,7 @@ model:
actor:
_target_: model.diffusion.mlp_diffusion.DiffusionMLP
horizon_steps: ${horizon_steps}
transition_dim: ${transition_dim}
action_dim: ${action_dim}
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
time_dim: 16
mlp_dims: [512, 512, 512]
cfg/gym/finetune/halfcheetah-v2/ft_ppo_diffusion_mlp.yaml
+3 -3
View File
@@ -6,15 +6,15 @@ hydra:
_target_: agent.finetune.train_ppo_diffusion_agent.TrainPPODiffusionAgent
name: ${env_name}_ppo_diffusion_mlp_ta${horizon_steps}_td${denoising_steps}_tdf${ft_denoising_steps}
logdir: ${oc.env:DPPO_LOG_DIR}/gym-finetune/${name}/${now:%Y-%m-%d}_${now:%H-%M-%S}
logdir: ${oc.env:DPPO_LOG_DIR}/gym-finetune/${name}/${now:%Y-%m-%d}_${now:%H-%M-%S}_${seed}
base_policy_path: ${oc.env:DPPO_LOG_DIR}/gym-pretrain/halfcheetah-medium-v2_pre_diffusion_mlp_ta4_td20/2024-06-12_23-04-42/checkpoint/state_3000.pt
normalization_path: ${oc.env:DPPO_DATA_DIR}/gym/${env_name}/normalization.npz
seed: 42
device: cuda:0
env_name: halfcheetah-medium-v2
obs_dim: 17
action_dim: 6
transition_dim: ${action_dim}
denoising_steps: 20
ft_denoising_steps: 10
cond_steps: 1
@@ -93,7 +93,7 @@ model:
residual_style: True
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
horizon_steps: ${horizon_steps}
transition_dim: ${transition_dim}
action_dim: ${action_dim}
critic:
_target_: model.common.critic.CriticObs
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
cfg/gym/finetune/halfcheetah-v2/ft_ppo_exact_diffusion_mlp.yaml
+3 -4
View File
@@ -6,15 +6,15 @@ hydra:
_target_: agent.finetune.train_ppo_exact_diffusion_agent.TrainPPOExactDiffusionAgent
name: ${env_name}_ppo_exact_diffusion_mlp_ta${horizon_steps}_td${denoising_steps}_tdf${ft_denoising_steps}
logdir: ${oc.env:DPPO_LOG_DIR}/gym-finetune/${name}/${now:%Y-%m-%d}_${now:%H-%M-%S}
logdir: ${oc.env:DPPO_LOG_DIR}/gym-finetune/${name}/${now:%Y-%m-%d}_${now:%H-%M-%S}_${seed}
base_policy_path: ${oc.env:DPPO_LOG_DIR}/gym-pretrain/halfcheetah-medium-v2_pre_diffusion_mlp_ta4_td20/2024-06-12_23-04-42/checkpoint/state_3000.pt
normalization_path: ${oc.env:DPPO_DATA_DIR}/gym/${env_name}/normalization.npz
seed: 42
device: cuda:0
env_name: halfcheetah-medium-v2
obs_dim: 17
action_dim: 6
transition_dim: ${action_dim}
denoising_steps: 20
ft_denoising_steps: 10
cond_steps: 1
@@ -87,7 +87,6 @@ model:
sde_min_beta: 1e-10
sde_probability_flow: True
#
gamma_denoising: 0.99
clip_ploss_coef: 0.01
min_sampling_denoising_std: 0.1
min_logprob_denoising_std: 0.1
@@ -101,7 +100,7 @@ model:
residual_style: True
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
horizon_steps: ${horizon_steps}
transition_dim: ${transition_dim}
action_dim: ${action_dim}
critic:
_target_: model.common.critic.CriticObs
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
cfg/gym/finetune/halfcheetah-v2/ft_qsm_diffusion_mlp.yaml
+8 -8
View File
@@ -6,15 +6,15 @@ hydra:
_target_: agent.finetune.train_qsm_diffusion_agent.TrainQSMDiffusionAgent
name: ${env_name}_qsm_diffusion_mlp_ta${horizon_steps}_td${denoising_steps}
logdir: ${oc.env:DPPO_LOG_DIR}/gym-finetune/${name}/${now:%Y-%m-%d}_${now:%H-%M-%S}
logdir: ${oc.env:DPPO_LOG_DIR}/gym-finetune/${name}/${now:%Y-%m-%d}_${now:%H-%M-%S}_${seed}
base_policy_path: ${oc.env:DPPO_LOG_DIR}/gym-pretrain/halfcheetah-medium-v2_pre_diffusion_mlp_ta4_td20/2024-06-12_23-04-42/checkpoint/state_3000.pt
normalization_path: ${oc.env:DPPO_DATA_DIR}/gym/${env_name}/normalization.npz
seed: 42
device: cuda:0
env_name: halfcheetah-medium-v2
obs_dim: 17
action_dim: 6
transition_dim: ${action_dim}
denoising_steps: 20
cond_steps: 1
horizon_steps: 4
@@ -65,11 +65,11 @@ train:
num: 0
# QSM specific
scale_reward_factor: 0.01
q_grad_coeff: 50
critic_tau: 0.005 # rate of target q network update
buffer_size: 5000
batch_size: 256
replay_ratio: 32
q_grad_coeff: 10
critic_tau: 0.005
buffer_size: 25000
replay_ratio: 16
batch_size: 1000
model:
_target_: model.diffusion.diffusion_qsm.QSMDiffusion
@@ -81,7 +81,7 @@ model:
actor:
_target_: model.diffusion.mlp_diffusion.DiffusionMLP
horizon_steps: ${horizon_steps}
transition_dim: ${transition_dim}
action_dim: ${action_dim}
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
time_dim: 16
mlp_dims: [512, 512, 512]
cfg/gym/finetune/halfcheetah-v2/ft_rwr_diffusion_mlp.yaml
+3 -3
View File
@@ -6,15 +6,15 @@ hydra:
_target_: agent.finetune.train_rwr_diffusion_agent.TrainRWRDiffusionAgent
name: ${env_name}_rwr_diffusion_mlp_ta${horizon_steps}_td${denoising_steps}
logdir: ${oc.env:DPPO_LOG_DIR}/gym-finetune/${name}/${now:%Y-%m-%d}_${now:%H-%M-%S}
logdir: ${oc.env:DPPO_LOG_DIR}/gym-finetune/${name}/${now:%Y-%m-%d}_${now:%H-%M-%S}_${seed}
base_policy_path: ${oc.env:DPPO_LOG_DIR}/gym-pretrain/halfcheetah-medium-v2_pre_diffusion_mlp_ta4_td20/2024-06-12_23-04-42/checkpoint/state_3000.pt
normalization_path: ${oc.env:DPPO_DATA_DIR}/gym/${env_name}/normalization.npz
seed: 42
device: cuda:0
env_name: halfcheetah-medium-v2
obs_dim: 17
action_dim: 6
transition_dim: ${action_dim}
denoising_steps: 20
cond_steps: 1
horizon_steps: 4
@@ -73,7 +73,7 @@ model:
network:
_target_: model.diffusion.mlp_diffusion.DiffusionMLP
horizon_steps: ${horizon_steps}
transition_dim: ${transition_dim}
action_dim: ${action_dim}
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
time_dim: 16
mlp_dims: [512, 512, 512]
cfg/gym/finetune/halfcheetah-v2/ibrl_mlp.yaml
+109
View File
@@ -0,0 +1,109 @@
defaults:
- _self_
hydra:
run:
dir: ${logdir}
_target_: agent.finetune.train_ibrl_agent.TrainIBRLAgent
name: ${env_name}_ibrl_mlp_ta${horizon_steps}
logdir: ${oc.env:DPPO_LOG_DIR}/gym-finetune/${name}/${now:%Y-%m-%d}_${now:%H-%M-%S}_${seed}
normalization_path: ${oc.env:DPPO_DATA_DIR}/gym/${env_name}/normalization.npz
base_policy_path:
offline_dataset_path: ${oc.env:DPPO_DATA_DIR}/gym/${env_name}/train.npz
seed: 42
device: cuda:0
env_name: halfcheetah-medium-v2
obs_dim: 17
action_dim: 6
cond_steps: 1
horizon_steps: 1
act_steps: 1
env:
n_envs: 1
name: ${env_name}
max_episode_steps: 1000
reset_at_iteration: False
save_video: False
best_reward_threshold_for_success: 3
wrappers:
mujoco_locomotion_lowdim:
normalization_path: ${normalization_path}
multi_step:
n_obs_steps: ${cond_steps}
n_action_steps: ${act_steps}
max_episode_steps: ${env.max_episode_steps}
reset_within_step: True
wandb:
entity: ${oc.env:DPPO_WANDB_ENTITY}
project: ibrl-${env_name}
run: ${now:%H-%M-%S}_${name}
train:
n_train_itr: 300000
n_steps: 1
gamma: 0.99
actor_lr: 1e-4
actor_weight_decay: 0
actor_lr_scheduler:
first_cycle_steps: 1000
warmup_steps: 10
min_lr: 1e-4
critic_lr: 1e-4
critic_weight_decay: 0
critic_lr_scheduler:
first_cycle_steps: 1000
warmup_steps: 10
min_lr: 1e-4
save_model_freq: 50000
val_freq: 2000
render:
freq: 1
num: 0
log_freq: 200
# IBRL specific
batch_size: 256
target_ema_rate: 0.01
scale_reward_factor: 1
critic_num_update: 5
buffer_size: 300000
n_eval_episode: 10
n_explore_steps: 0
update_freq: 2
model:
_target_: model.rl.gaussian_ibrl.IBRL_Gaussian
randn_clip_value: 3
n_critics: 5
soft_action_sample: True
soft_action_sample_beta: 10
network_path: ${base_policy_path}
actor:
_target_: model.common.mlp_gaussian.Gaussian_MLP
mlp_dims: [256, 256, 256]
activation_type: Mish
fixed_std: 0.1
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
horizon_steps: ${horizon_steps}
critic:
_target_: model.common.critic.CriticObsAct
mlp_dims: [256, 256, 256]
activation_type: ReLU
use_layernorm: True
double_q: False # use ensemble
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
action_dim: ${action_dim}
action_steps: ${act_steps}
horizon_steps: ${horizon_steps}
device: ${device}
offline_dataset:
_target_: agent.dataset.sequence.StitchedSequenceQLearningDataset
dataset_path: ${offline_dataset_path}
horizon_steps: ${horizon_steps}
cond_steps: ${cond_steps}
device: ${device}
max_n_episodes: 50
cfg/gym/finetune/halfcheetah-v2/ppo_diffusion_mlp.yaml
+3 -3
View File
@@ -6,14 +6,14 @@ hydra:
_target_: agent.finetune.train_ppo_diffusion_agent.TrainPPODiffusionAgent
name: ${env_name}_nopre_ppo_diffusion_mlp_ta${horizon_steps}_td${denoising_steps}
logdir: ${oc.env:DPPO_LOG_DIR}/gym-finetune/${name}/${now:%Y-%m-%d}_${now:%H-%M-%S}
logdir: ${oc.env:DPPO_LOG_DIR}/gym-finetune/${name}/${now:%Y-%m-%d}_${now:%H-%M-%S}_${seed}
normalization_path: ${oc.env:DPPO_DATA_DIR}/gym/${env_name}/normalization.npz
seed: 42
device: cuda:0
env_name: halfcheetah-medium-v2
obs_dim: 17
action_dim: 6
transition_dim: ${action_dim}
denoising_steps: 20
ft_denoising_steps: 20
cond_steps: 1
@@ -86,7 +86,7 @@ model:
actor:
_target_: model.diffusion.mlp_diffusion.DiffusionMLP
horizon_steps: ${horizon_steps}
transition_dim: ${transition_dim}
action_dim: ${action_dim}
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
time_dim: 16
mlp_dims: [512, 512, 512]
cfg/gym/finetune/halfcheetah-v2/ppo_gaussian_mlp.yaml
+4 -4
View File
@@ -6,14 +6,14 @@ hydra:
_target_: agent.finetune.train_ppo_gaussian_agent.TrainPPOGaussianAgent
name: ${env_name}_nopre_ppo_gaussian_mlp_ta${horizon_steps}
logdir: ${oc.env:DPPO_LOG_DIR}/gym-finetune/${name}/${now:%Y-%m-%d}_${now:%H-%M-%S}
logdir: ${oc.env:DPPO_LOG_DIR}/gym-finetune/${name}/${now:%Y-%m-%d}_${now:%H-%M-%S}_${seed}
normalization_path: ${oc.env:DPPO_DATA_DIR}/gym/${env_name}/normalization.npz
seed: 42
device: cuda:0
env_name: halfcheetah-medium-v2
obs_dim: 17
action_dim: 6
transition_dim: ${action_dim}
cond_steps: 1
horizon_steps: 1
act_steps: 1
@@ -79,10 +79,10 @@ model:
_target_: model.common.mlp_gaussian.Gaussian_MLP
mlp_dims: [512, 512, 512]
activation_type: ReLU
residual_style: True
residual_style: False # with new logvar head
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
horizon_steps: ${horizon_steps}
transition_dim: ${transition_dim}
action_dim: ${action_dim}
critic:
_target_: model.common.critic.CriticObs
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
cfg/gym/finetune/halfcheetah-v2/rlpd_mlp.yaml
+109
View File
@@ -0,0 +1,109 @@
defaults:
- _self_
hydra:
run:
dir: ${logdir}
_target_: agent.finetune.train_rlpd_agent.TrainRLPDAgent
name: ${env_name}_rlpd_mlp_ta${horizon_steps}
logdir: ${oc.env:DPPO_LOG_DIR}/gym-finetune/${name}/${now:%Y-%m-%d}_${now:%H-%M-%S}_${seed}
normalization_path: ${oc.env:DPPO_DATA_DIR}/gym/${env_name}/normalization.npz
offline_dataset_path: ${oc.env:DPPO_DATA_DIR}/gym/${env_name}/train.npz
seed: 42
device: cuda:0
env_name: halfcheetah-medium-v2
obs_dim: 17
action_dim: 6
cond_steps: 1
horizon_steps: 1
act_steps: 1
env:
n_envs: 1
name: ${env_name}
max_episode_steps: 1000
reset_at_iteration: False
save_video: False
best_reward_threshold_for_success: 3
wrappers:
mujoco_locomotion_lowdim:
normalization_path: ${normalization_path}
multi_step:
n_obs_steps: ${cond_steps}
n_action_steps: ${act_steps}
max_episode_steps: ${env.max_episode_steps}
reset_within_step: True
wandb:
entity: ${oc.env:DPPO_WANDB_ENTITY}
project: rlpd-${env_name}
run: ${now:%H-%M-%S}_${name}
train:
n_train_itr: 250000
n_steps: 1
gamma: 0.99
actor_lr: 3e-4
actor_weight_decay: 0
actor_lr_scheduler:
first_cycle_steps: 1000
warmup_steps: 10
min_lr: 3e-4
critic_lr: 3e-4
critic_weight_decay: 0
critic_lr_scheduler:
first_cycle_steps: 1000
warmup_steps: 10
min_lr: 3e-4
save_model_freq: 50000
val_freq: 5000
render:
freq: 1
num: 0
log_freq: 200
# RLPD specific
batch_size: 256
target_ema_rate: 0.005
scale_reward_factor: 1
critic_num_update: 20
buffer_size: 1000000
n_eval_episode: 10
n_explore_steps: 5000
target_entropy: ${eval:'- ${action_dim} * ${act_steps}'}
init_temperature: 1
model:
_target_: model.rl.gaussian_rlpd.RLPD_Gaussian
randn_clip_value: 10
tanh_output: True # squash after sampling
backup_entropy: True
n_critics: 10 # Ensemble size for critic models
actor:
_target_: model.common.mlp_gaussian.Gaussian_MLP
mlp_dims: [256, 256]
activation_type: ReLU
tanh_output: False # squash after sampling instead
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
horizon_steps: ${horizon_steps}
std_max: 7.3891
std_min: 0.0067
critic:
_target_: model.common.critic.CriticObsAct
mlp_dims: [256, 256]
activation_type: ReLU
use_layernorm: True
double_q: False # use ensemble
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
action_dim: ${action_dim}
action_steps: ${act_steps}
horizon_steps: ${horizon_steps}
device: ${device}
offline_dataset:
_target_: agent.dataset.sequence.StitchedSequenceQLearningDataset
dataset_path: ${offline_dataset_path}
horizon_steps: ${horizon_steps}
cond_steps: ${cond_steps}
device: ${device}
cfg/gym/finetune/halfcheetah-v2/sac_mlp.yaml
+89
View File
@@ -0,0 +1,89 @@
defaults:
- _self_
hydra:
run:
dir: ${logdir}
_target_: agent.finetune.train_sac_agent.TrainSACAgent
name: ${env_name}_sac_mlp_ta${horizon_steps}
logdir: ${oc.env:DPPO_LOG_DIR}/gym-finetune/${name}/${now:%Y-%m-%d}_${now:%H-%M-%S}_${seed}
normalization_path: ${oc.env:DPPO_DATA_DIR}/gym/${env_name}/normalization.npz
offline_dataset_path: ${oc.env:DPPO_DATA_DIR}/gym/${env_name}/train.npz
seed: 42
device: cuda:0
env_name: halfcheetah-medium-v2
obs_dim: 17
action_dim: 6
cond_steps: 1
horizon_steps: 1
act_steps: 1
env:
n_envs: 1
name: ${env_name}
max_episode_steps: 1000
reset_at_iteration: False
save_video: False
best_reward_threshold_for_success: 3
wrappers:
mujoco_locomotion_lowdim:
normalization_path: ${normalization_path}
multi_step:
n_obs_steps: ${cond_steps}
n_action_steps: ${act_steps}
max_episode_steps: ${env.max_episode_steps}
reset_within_step: True
wandb:
entity: ${oc.env:DPPO_WANDB_ENTITY}
project: sac-gym-${env_name}
run: ${now:%H-%M-%S}_${name}
train:
n_train_itr: 1000000
n_steps: 1
gamma: 0.99
actor_lr: 3e-4
critic_lr: 1e-3
save_model_freq: 100000
val_freq: 10000
render:
freq: 1
num: 0
log_freq: 200
# SAC specific
batch_size: 256
target_ema_rate: 0.005
scale_reward_factor: 1
critic_replay_ratio: 256
actor_replay_ratio: 128
buffer_size: 1000000
n_eval_episode: 10
n_explore_steps: 5000
target_entropy: ${eval:'- ${action_dim} * ${act_steps}'}
init_temperature: 1
model:
_target_: model.rl.gaussian_sac.SAC_Gaussian
randn_clip_value: 10
tanh_output: True # squash after sampling
actor:
_target_: model.common.mlp_gaussian.Gaussian_MLP
mlp_dims: [256, 256]
activation_type: ReLU
tanh_output: False # squash after sampling instead
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
horizon_steps: ${horizon_steps}
std_max: 7.3891
std_min: 0.0067
critic: # no layernorm
_target_: model.common.critic.CriticObsAct
mlp_dims: [256, 256]
activation_type: ReLU
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
action_dim: ${action_dim}
action_steps: ${act_steps}
horizon_steps: ${horizon_steps}
device: ${device}