dodox/dppo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

v0.6 (#18)

Browse Source 
* Sampling over both env and denoising steps in DPPO updates (#13)

* sample one from each chain

* full random sampling

* Add Proficient Human (PH) Configs and Pipeline (#16)

* fix missing cfg

* add ph config

* fix how terminated flags are added to buffer in ibrl

* add ph config

* offline calql for 1M gradient updates

* bug fix: number of calql online gradient steps is the number of new transitions collected

* add sample config for DPPO with ta=1

* Sampling over both env and denoising steps in DPPO updates (#13)

* sample one from each chain

* full random sampling

* fix diffusion loss when predicting initial noise

* fix dppo inds

* fix typo

* remove print statement

---------

Co-authored-by: Justin M. Lidard <jlidard@neuronic.cs.princeton.edu>
Co-authored-by: allenzren <allen.ren@princeton.edu>

* update robomimic configs

* better calql formulation

* optimize calql and ibrl training

* optimize data transfer in ppo agents

* add kitchen configs

* re-organize config folders, rerun calql and rlpd

* add scratch gym locomotion configs

* add kitchen installation dependencies

* use truncated for termination in furniture env

* update furniture and gym configs

* update README and dependencies with kitchen

* add url for new data and checkpoints

* update demo RL configs

* update batch sizes for furniture unet configs

* raise error about dropout in residual mlp

* fix observation bug in bc loss

---------

Co-authored-by: Justin Lidard <60638575+jlidard@users.noreply.github.com>
Co-authored-by: Justin M. Lidard <jlidard@neuronic.cs.princeton.edu>
This commit is contained in:
Allen Z. Ren
2024-10-30 19:58:06 -04:00
committed by GitHub
co-authored by Justin M. Lidard Justin Lidard
parent 7b10df690d
commit dc8e0c9edc
126 changed files with 4614 additions and 553 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cfg/robomimic/finetune/square/calql_mlp_online.yaml
+2 -2
View File
@@ -7,7 +7,7 @@ _target_: agent.finetune.train_calql_agent.TrainCalQLAgent
name: ${env_name}_calql_mlp_ta${horizon_steps}
logdir: ${oc.env:DPPO_LOG_DIR}/robomimic-finetune/${name}/${now:%Y-%m-%d}_${now:%H-%M-%S}_${seed}
base_policy_path:
base_policy_path: ${oc.env:DPPO_LOG_DIR}/robomimic-pretrain/square_calql_mlp_ta1/2024-10-25_22-44-12_42/checkpoint/state_999.pt
robomimic_env_cfg_path: cfg/robomimic/env_meta/${env_name}.json
normalization_path: ${oc.env:DPPO_DATA_DIR}/robomimic/${env_name}/normalization.npz
offline_dataset_path: ${oc.env:DPPO_DATA_DIR}/robomimic/${env_name}/train.npz
@@ -97,7 +97,7 @@ model:
tanh_output: False # squash after sampling instead
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
horizon_steps: ${horizon_steps}
action_dim: ${action_dim}
std_max: 7.3891
std_min: 0.0067
critic:
cfg/robomimic/finetune/square/calql_mlp_online_ph.yaml
+122
View File
@@ -0,0 +1,122 @@
defaults:
- _self_
hydra:
run:
dir: ${logdir}
_target_: agent.finetune.train_calql_agent.TrainCalQLAgent
name: ${env_name}_calql_mlp_ta${horizon_steps}
logdir: ${oc.env:DPPO_LOG_DIR}/robomimic-finetune/${name}/${now:%Y-%m-%d}_${now:%H-%M-%S}_${seed}
base_policy_path: ${oc.env:DPPO_LOG_DIR}/robomimic-pretrain/square_calql_mlp_ta1/2024-10-09_11-05-07_0/checkpoint/state_999.pt
robomimic_env_cfg_path: cfg/robomimic/env_meta/${env_name}.json
normalization_path: ${oc.env:DPPO_DATA_DIR}/robomimic/${env_name}-ph/normalization.npz
offline_dataset_path: ${oc.env:DPPO_DATA_DIR}/robomimic/${env_name}-ph/train.npz
seed: 42
device: cuda:0
env_name: square
obs_dim: 23
action_dim: 7
cond_steps: 1
horizon_steps: 1
act_steps: 1
env:
n_envs: 1
name: ${env_name}
best_reward_threshold_for_success: 1
max_episode_steps: 400
reset_at_iteration: False
save_video: False
wrappers:
robomimic_lowdim:
normalization_path: ${normalization_path}
low_dim_keys: ['robot0_eef_pos',
'robot0_eef_quat',
'robot0_gripper_qpos',
'object'] # same order of preprocessed observations
multi_step:
n_obs_steps: ${cond_steps}
n_action_steps: ${act_steps}
max_episode_steps: ${env.max_episode_steps}
reset_within_step: True
wandb:
entity: ${oc.env:DPPO_WANDB_ENTITY}
project: calql-${env_name}
run: ${now:%H-%M-%S}_${name}
train:
n_train_itr: 10000
n_steps: 1 # not used
n_episode_per_epoch: 1
gamma: 0.99
actor_lr: 1e-4
actor_weight_decay: 0
actor_lr_scheduler:
first_cycle_steps: 1000
warmup_steps: 10
min_lr: 1e-4
critic_lr: 3e-4
critic_weight_decay: 0
critic_lr_scheduler:
first_cycle_steps: 1000
warmup_steps: 10
min_lr: 3e-4
save_model_freq: 100
val_freq: 10
render:
freq: 1
num: 0
log_freq: 1
# CalQL specific
train_online: True
batch_size: 256
n_random_actions: 4
target_ema_rate: 0.005
scale_reward_factor: 1.0
num_update: 1000
buffer_size: 1000000
online_utd_ratio: 1
n_eval_episode: 40
n_explore_steps: 0
target_entropy: ${eval:'- ${action_dim} * ${act_steps}'}
init_temperature: 1
automatic_entropy_tuning: True
model:
_target_: model.rl.gaussian_calql.CalQL_Gaussian
randn_clip_value: 3
cql_min_q_weight: 5.0
tanh_output: True
network_path: ${base_policy_path}
actor:
_target_: model.common.mlp_gaussian.Gaussian_MLP
mlp_dims: [512, 512, 512]
activation_type: ReLU
tanh_output: False # squash after sampling instead
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
horizon_steps: ${horizon_steps}
action_dim: ${action_dim}
std_max: 7.3891
std_min: 0.0067
critic:
_target_: model.common.critic.CriticObsAct
mlp_dims: [256, 256, 256]
activation_type: ReLU
use_layernorm: True
double_q: True
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
action_dim: ${action_dim}
action_steps: ${act_steps}
horizon_steps: ${horizon_steps}
device: ${device}
offline_dataset:
_target_: agent.dataset.sequence.StitchedSequenceQLearningDataset
dataset_path: ${offline_dataset_path}
horizon_steps: ${horizon_steps}
cond_steps: ${cond_steps}
device: ${device}
discount_factor: ${train.gamma}
get_mc_return: True
cfg/robomimic/finetune/square/ft_awr_diffusion_mlp.yaml
+1 -1
View File
@@ -46,7 +46,7 @@ wandb:
run: ${now:%H-%M-%S}_${name}
train:
n_train_itr: 300
n_train_itr: 201
n_critic_warmup_itr: 2
n_steps: 400
gamma: 0.999
cfg/robomimic/finetune/square/ft_dql_diffusion_mlp.yaml
+1 -1
View File
@@ -46,7 +46,7 @@ wandb:
run: ${now:%H-%M-%S}_${name}
train:
n_train_itr: 300
n_train_itr: 201
n_critic_warmup_itr: 5
n_steps: 400
gamma: 0.999
cfg/robomimic/finetune/square/ft_idql_diffusion_mlp.yaml
+1 -1
View File
@@ -46,7 +46,7 @@ wandb:
run: ${now:%H-%M-%S}_${name}
train:
n_train_itr: 300
n_train_itr: 201
n_critic_warmup_itr: 5
n_steps: 400
gamma: 0.999
cfg/robomimic/finetune/square/ft_ppo_diffusion_mlp.yaml
+4 -4
View File
@@ -47,16 +47,16 @@ wandb:
run: ${now:%H-%M-%S}_${name}
train:
n_train_itr: 500
n_train_itr: 201
n_critic_warmup_itr: 2
n_steps: 400
gamma: 0.999
actor_lr: 1e-5
actor_lr: 1e-4
actor_weight_decay: 0
actor_lr_scheduler:
first_cycle_steps: 1000
warmup_steps: 10
min_lr: 1e-5
min_lr: 1e-4
critic_lr: 1e-3
critic_weight_decay: 0
critic_lr_scheduler:
@@ -100,10 +100,10 @@ model:
action_dim: ${action_dim}
critic:
_target_: model.common.critic.CriticObs
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
mlp_dims: [256, 256, 256]
activation_type: Mish
residual_style: True
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
ft_denoising_steps: ${ft_denoising_steps}
horizon_steps: ${horizon_steps}
obs_dim: ${obs_dim}
cfg/robomimic/finetune/square/ft_ppo_diffusion_mlp_img.yaml
+6 -6
View File
@@ -1,7 +1,7 @@
defaults:
- _self_
hydra:
run:
run:
dir: ${logdir}
_target_: agent.finetune.train_ppo_diffusion_img_agent.TrainPPOImgDiffusionAgent
@@ -60,7 +60,7 @@ wandb:
run: ${now:%H-%M-%S}_${name}
train:
n_train_itr: 500
n_train_itr: 301
n_critic_warmup_itr: 2
n_steps: 400
gamma: 0.999
@@ -69,13 +69,13 @@ train:
actor_lr: 1e-5
actor_weight_decay: 0
actor_lr_scheduler:
first_cycle_steps: 500
first_cycle_steps: 1000
warmup_steps: 10
min_lr: 1e-5
critic_lr: 1e-3
critic_weight_decay: 0
critic_lr_scheduler:
first_cycle_steps: 500
first_cycle_steps: 1000
warmup_steps: 10
min_lr: 1e-3
save_model_freq: 100
@@ -96,7 +96,7 @@ train:
model:
_target_: model.diffusion.diffusion_ppo.PPODiffusion
# HP to tune
gamma_denoising: 0.9
gamma_denoising: 0.99
clip_ploss_coef: 0.01
clip_ploss_coef_base: 0.001
clip_ploss_coef_rate: 3
@@ -158,10 +158,10 @@ model:
embed_style: embed2
embed_norm: 0
img_cond_steps: ${img_cond_steps}
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
mlp_dims: [256, 256, 256]
activation_type: Mish
residual_style: True
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
ft_denoising_steps: ${ft_denoising_steps}
horizon_steps: ${horizon_steps}
obs_dim: ${obs_dim}
cfg/robomimic/finetune/square/ft_ppo_diffusion_mlp_ta1.yaml
+112
View File
@@ -0,0 +1,112 @@
defaults:
- _self_
hydra:
run:
dir: ${logdir}
_target_: agent.finetune.train_ppo_diffusion_agent.TrainPPODiffusionAgent
name: ${env_name}_ft_diffusion_mlp_ta${horizon_steps}_td${denoising_steps}_tdf${ft_denoising_steps}
logdir: ${oc.env:DPPO_LOG_DIR}/robomimic-finetune/${name}/${now:%Y-%m-%d}_${now:%H-%M-%S}_${seed}
base_policy_path: ${oc.env:DPPO_LOG_DIR}/robomimic-pretrain/square_pre_diffusion_mlp_ta1_td20/2024-09-29_02-14-14_42/checkpoint/state_8000.pt
robomimic_env_cfg_path: cfg/robomimic/env_meta/${env_name}.json
normalization_path: ${oc.env:DPPO_DATA_DIR}/robomimic/${env_name}/normalization.npz
seed: 42
device: cuda:0
env_name: square
obs_dim: 23
action_dim: 7
denoising_steps: 20
ft_denoising_steps: 10
cond_steps: 1
horizon_steps: 1
act_steps: 1
env:
n_envs: 50
name: ${env_name}
best_reward_threshold_for_success: 1
max_episode_steps: 400
save_video: false
wrappers:
robomimic_lowdim:
normalization_path: ${normalization_path}
low_dim_keys: ['robot0_eef_pos',
'robot0_eef_quat',
'robot0_gripper_qpos',
'object'] # same order of preprocessed observations
multi_step:
n_obs_steps: ${cond_steps}
n_action_steps: ${act_steps}
max_episode_steps: ${env.max_episode_steps}
reset_within_step: True
wandb:
entity: ${oc.env:DPPO_WANDB_ENTITY}
project: robomimic-${env_name}-finetune
run: ${now:%H-%M-%S}_${name}
train:
n_train_itr: 301
n_critic_warmup_itr: 2
n_steps: 400
gamma: 0.999
actor_lr: 1e-4
actor_weight_decay: 0
actor_lr_scheduler:
first_cycle_steps: 1000
warmup_steps: 10
min_lr: 1e-4
critic_lr: 1e-3
critic_weight_decay: 0
critic_lr_scheduler:
first_cycle_steps: 1000
warmup_steps: 10
min_lr: 1e-3
save_model_freq: 100
val_freq: 10
render:
freq: 1
num: 0
# PPO specific
reward_scale_running: True
reward_scale_const: 1.0
gae_lambda: 0.95
batch_size: 20000
update_epochs: 10
vf_coef: 0.5
target_kl: 1
model:
_target_: model.diffusion.diffusion_ppo.PPODiffusion
# HP to tune
gamma_denoising: 0.99
clip_ploss_coef: 0.01
clip_ploss_coef_base: 0.001
clip_ploss_coef_rate: 3
randn_clip_value: 3
min_sampling_denoising_std: 0.1
min_logprob_denoising_std: 0.1
#
network_path: ${base_policy_path}
actor:
_target_: model.diffusion.mlp_diffusion.DiffusionMLP
time_dim: 32
mlp_dims: [1024, 1024, 1024]
cond_mlp_dims: [512, 64]
residual_style: True
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
horizon_steps: ${horizon_steps}
action_dim: ${action_dim}
critic:
_target_: model.common.critic.CriticObs
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
mlp_dims: [256, 256, 256]
activation_type: Mish
residual_style: True
ft_denoising_steps: ${ft_denoising_steps}
horizon_steps: ${horizon_steps}
obs_dim: ${obs_dim}
action_dim: ${action_dim}
denoising_steps: ${denoising_steps}
device: ${device}
cfg/robomimic/finetune/square/ft_ppo_diffusion_mlp_ta1_ph.yaml
+112
View File
@@ -0,0 +1,112 @@
defaults:
- _self_
hydra:
run:
dir: ${logdir}
_target_: agent.finetune.train_ppo_diffusion_agent.TrainPPODiffusionAgent
name: ${env_name}_ft_diffusion_mlp_ta${horizon_steps}_td${denoising_steps}_tdf${ft_denoising_steps}
logdir: ${oc.env:DPPO_LOG_DIR}/robomimic-finetune/${name}/${now:%Y-%m-%d}_${now:%H-%M-%S}_${seed}
base_policy_path: ${oc.env:DPPO_LOG_DIR}/robomimic-pretrain/square_pre_diffusion_mlp_ta1_td20/2024-10-14_10-54-33_0/checkpoint/state_5000.pt
robomimic_env_cfg_path: cfg/robomimic/env_meta/${env_name}.json
normalization_path: ${oc.env:DPPO_DATA_DIR}/robomimic/${env_name}-ph/normalization.npz
seed: 42
device: cuda:0
env_name: square
obs_dim: 23
action_dim: 7
denoising_steps: 20
ft_denoising_steps: 10
cond_steps: 1
horizon_steps: 1
act_steps: 1
env:
n_envs: 40
name: ${env_name}
best_reward_threshold_for_success: 1
max_episode_steps: 400
save_video: false
wrappers:
robomimic_lowdim:
normalization_path: ${normalization_path}
low_dim_keys: ['robot0_eef_pos',
'robot0_eef_quat',
'robot0_gripper_qpos',
'object'] # same order of preprocessed observations
multi_step:
n_obs_steps: ${cond_steps}
n_action_steps: ${act_steps}
max_episode_steps: ${env.max_episode_steps}
reset_within_step: True
wandb:
entity: ${oc.env:DPPO_WANDB_ENTITY}
project: robomimic-${env_name}-finetune
run: ${now:%H-%M-%S}_${name}
train:
n_train_itr: 301
n_critic_warmup_itr: 2
n_steps: 400
gamma: 0.999
actor_lr: 1e-4
actor_weight_decay: 0
actor_lr_scheduler:
first_cycle_steps: 1000
warmup_steps: 10
min_lr: 1e-4
critic_lr: 1e-3
critic_weight_decay: 0
critic_lr_scheduler:
first_cycle_steps: 1000
warmup_steps: 10
min_lr: 1e-3
save_model_freq: 100
val_freq: 10
render:
freq: 1
num: 0
# PPO specific
reward_scale_running: True
reward_scale_const: 1.0
gae_lambda: 0.95
batch_size: 8000
update_epochs: 10
vf_coef: 0.5
target_kl: 1
model:
_target_: model.diffusion.diffusion_ppo.PPODiffusion
# HP to tune
gamma_denoising: 0.9
clip_ploss_coef: 0.01
clip_ploss_coef_base: 0.001
clip_ploss_coef_rate: 3
randn_clip_value: 3
min_sampling_denoising_std: 0.1
min_logprob_denoising_std: 0.1
#
network_path: ${base_policy_path}
actor:
_target_: model.diffusion.mlp_diffusion.DiffusionMLP
time_dim: 32
mlp_dims: [1024, 1024, 1024]
cond_mlp_dims: [512, 64]
residual_style: True
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
horizon_steps: ${horizon_steps}
action_dim: ${action_dim}
critic:
_target_: model.common.critic.CriticObs
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
mlp_dims: [256, 256, 256]
activation_type: Mish
residual_style: True
ft_denoising_steps: ${ft_denoising_steps}
horizon_steps: ${horizon_steps}
obs_dim: ${obs_dim}
action_dim: ${action_dim}
denoising_steps: ${denoising_steps}
device: ${device}
cfg/robomimic/finetune/square/ft_qsm_diffusion_mlp.yaml
+1 -1
View File
@@ -46,7 +46,7 @@ wandb:
run: ${now:%H-%M-%S}_${name}
train:
n_train_itr: 300
n_train_itr: 201
n_critic_warmup_itr: 5
n_steps: 400
gamma: 0.999
cfg/robomimic/finetune/square/ft_rwr_diffusion_mlp.yaml
+1 -1
View File
@@ -46,7 +46,7 @@ wandb:
run: ${now:%H-%M-%S}_${name}
train:
n_train_itr: 300
n_train_itr: 201
n_critic_warmup_itr: 2
n_steps: 400
gamma: 0.999
cfg/robomimic/finetune/square/ibrl_mlp.yaml
+2 -2
View File
@@ -7,7 +7,7 @@ _target_: agent.finetune.train_ibrl_agent.TrainIBRLAgent
name: ${env_name}_ibrl_mlp_ta${horizon_steps}
logdir: ${oc.env:DPPO_LOG_DIR}/robomimic-finetune/${name}/${now:%Y-%m-%d}_${now:%H-%M-%S}_${seed}
base_policy_path:
base_policy_path: ${oc.env:DPPO_LOG_DIR}/robomimic-pretrain/square_pre_gaussian_mlp_ta1/2024-09-28_13-42-43_42/checkpoint/state_5000.pt
robomimic_env_cfg_path: cfg/robomimic/env_meta/${env_name}.json
normalization_path: ${oc.env:DPPO_DATA_DIR}/robomimic/${env_name}/normalization.npz
offline_dataset_path: ${oc.env:DPPO_DATA_DIR}/robomimic/${env_name}/train.npz
@@ -93,7 +93,7 @@ model:
fixed_std: 0.1
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
horizon_steps: ${horizon_steps}
action_dim: ${action_dim}
critic:
_target_: model.common.critic.CriticObsAct
mlp_dims: [1024, 1024, 1024]
cfg/robomimic/finetune/square/rlpd_mlp.yaml → cfg/robomimic/finetune/square/ibrl_mlp_ph.yaml
+21 -20
View File
@@ -3,13 +3,14 @@ defaults:
hydra:
run:
dir: ${logdir}
_target_: agent.finetune.train_rlpd_agent.TrainRLPDAgent
_target_: agent.finetune.train_ibrl_agent.TrainIBRLAgent
name: ${env_name}_rlpd_mlp_ta${horizon_steps}
name: ${env_name}_ibrl_mlp_ta${horizon_steps}
logdir: ${oc.env:DPPO_LOG_DIR}/robomimic-finetune/${name}/${now:%Y-%m-%d}_${now:%H-%M-%S}_${seed}
base_policy_path: ${oc.env:DPPO_LOG_DIR}/robomimic-pretrain/square_pre_gaussian_mlp_ta1/2024-10-08_20-52-42_0/checkpoint/state_5000.pt
robomimic_env_cfg_path: cfg/robomimic/env_meta/${env_name}.json
normalization_path: ${oc.env:DPPO_DATA_DIR}/robomimic/${env_name}/normalization.npz
offline_dataset_path: ${oc.env:DPPO_DATA_DIR}/robomimic/${env_name}/train.npz
normalization_path: ${oc.env:DPPO_DATA_DIR}/robomimic/${env_name}-ph/normalization.npz
offline_dataset_path: ${oc.env:DPPO_DATA_DIR}/robomimic/${env_name}-ph/train.npz
seed: 42
device: cuda:0
@@ -23,7 +24,7 @@ act_steps: 1
env:
n_envs: 1
name: ${env_name}
max_episode_steps: 400
max_episode_steps: 350 # IBRL uses 300
reset_at_iteration: False
save_video: False
best_reward_threshold_for_success: 1
@@ -42,7 +43,7 @@ env:
wandb:
entity: ${oc.env:DPPO_WANDB_ENTITY}
project: rlpd-${env_name}
project: ibrl-${env_name}
run: ${now:%H-%M-%S}_${name}
train:
@@ -67,7 +68,7 @@ train:
freq: 10000
num: 0
log_freq: 200
# RLPD specific
# IBRL specific
batch_size: 256
target_ema_rate: 0.01
scale_reward_factor: 1
@@ -75,28 +76,27 @@ train:
buffer_size: 400000
n_eval_episode: 40
n_explore_steps: 0
target_entropy: ${eval:'- ${action_dim} * ${act_steps}'}
init_temperature: 1
update_freq: 2
model:
_target_: model.rl.gaussian_rlpd.RLPD_Gaussian
randn_clip_value: 10
backup_entropy: True
_target_: model.rl.gaussian_ibrl.IBRL_Gaussian
randn_clip_value: 3
n_critics: 5
tanh_output: True
soft_action_sample: True
soft_action_sample_beta: 10
network_path: ${base_policy_path}
actor:
_target_: model.common.mlp_gaussian.Gaussian_MLP
mlp_dims: [512, 512, 512]
mlp_dims: [1024, 1024, 1024]
activation_type: ReLU
tanh_output: False # squash after sampling instead
dropout: 0.5
fixed_std: 0.1
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
horizon_steps: ${horizon_steps}
std_max: 7.3891
std_min: 0.0067
action_dim: ${action_dim}
critic:
_target_: model.common.critic.CriticObsAct
mlp_dims: [256, 256, 256]
mlp_dims: [1024, 1024, 1024]
activation_type: ReLU
use_layernorm: True
double_q: False # use ensemble
@@ -111,4 +111,5 @@ offline_dataset:
dataset_path: ${offline_dataset_path}
horizon_steps: ${horizon_steps}
cond_steps: ${cond_steps}
device: ${device}
device: ${device}
max_n_episodes: 100