dodox/dppo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

v0.6 (#18)

Browse Source 
* Sampling over both env and denoising steps in DPPO updates (#13)

* sample one from each chain

* full random sampling

* Add Proficient Human (PH) Configs and Pipeline (#16)

* fix missing cfg

* add ph config

* fix how terminated flags are added to buffer in ibrl

* add ph config

* offline calql for 1M gradient updates

* bug fix: number of calql online gradient steps is the number of new transitions collected

* add sample config for DPPO with ta=1

* Sampling over both env and denoising steps in DPPO updates (#13)

* sample one from each chain

* full random sampling

* fix diffusion loss when predicting initial noise

* fix dppo inds

* fix typo

* remove print statement

---------

Co-authored-by: Justin M. Lidard <jlidard@neuronic.cs.princeton.edu>
Co-authored-by: allenzren <allen.ren@princeton.edu>

* update robomimic configs

* better calql formulation

* optimize calql and ibrl training

* optimize data transfer in ppo agents

* add kitchen configs

* re-organize config folders, rerun calql and rlpd

* add scratch gym locomotion configs

* add kitchen installation dependencies

* use truncated for termination in furniture env

* update furniture and gym configs

* update README and dependencies with kitchen

* add url for new data and checkpoints

* update demo RL configs

* update batch sizes for furniture unet configs

* raise error about dropout in residual mlp

* fix observation bug in bc loss

---------

Co-authored-by: Justin Lidard <60638575+jlidard@users.noreply.github.com>
Co-authored-by: Justin M. Lidard <jlidard@neuronic.cs.princeton.edu>
This commit is contained in:
Allen Z. Ren
2024-10-30 19:58:06 -04:00
committed by GitHub
co-authored by Justin M. Lidard Justin Lidard
parent 7b10df690d
commit dc8e0c9edc
126 changed files with 4614 additions and 553 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cfg/robomimic/pretrain/can/calql_mlp_offline.yaml
+5 -5
View File
@@ -46,7 +46,7 @@ wandb:
run: ${now:%H-%M-%S}_${name}
train:
n_train_itr: 100
n_train_itr: 1000
n_steps: 1
gamma: 0.99
actor_lr: 1e-4
@@ -61,8 +61,8 @@ train:
first_cycle_steps: 1000
warmup_steps: 10
min_lr: 3e-4
save_model_freq: 10
val_freq: 10
save_model_freq: 100
val_freq: 20
render:
freq: 1
num: 0
@@ -70,7 +70,7 @@ train:
# CalQL specific
train_online: False
batch_size: 256
n_random_actions: 4
n_random_actions: 10
target_ema_rate: 0.005
scale_reward_factor: 1.0
num_update: 1000
@@ -93,7 +93,7 @@ model:
tanh_output: False # squash after sampling instead
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
horizon_steps: ${horizon_steps}
action_dim: ${action_dim}
std_max: 7.3891
std_min: 0.0067
critic:
cfg/robomimic/pretrain/can/calql_mlp_offline_ph.yaml
+118
View File
@@ -0,0 +1,118 @@
defaults:
- _self_
hydra:
run:
dir: ${logdir}
_target_: agent.finetune.train_calql_agent.TrainCalQLAgent
name: ${env_name}_calql_mlp_ta${horizon_steps}
logdir: ${oc.env:DPPO_LOG_DIR}/robomimic-pretrain/${name}/${now:%Y-%m-%d}_${now:%H-%M-%S}_${seed}
robomimic_env_cfg_path: cfg/robomimic/env_meta/${env_name}.json
normalization_path: ${oc.env:DPPO_DATA_DIR}/robomimic/${env_name}-ph/normalization.npz
offline_dataset_path: ${oc.env:DPPO_DATA_DIR}/robomimic/${env_name}-ph/train.npz
seed: 42
device: cuda:0
env_name: can
obs_dim: 23
action_dim: 7
cond_steps: 1
horizon_steps: 1
act_steps: 1
env:
n_envs: 1
name: ${env_name}
best_reward_threshold_for_success: 1
max_episode_steps: 300
reset_at_iteration: False
save_video: False
wrappers:
robomimic_lowdim:
normalization_path: ${normalization_path}
low_dim_keys: ['robot0_eef_pos',
'robot0_eef_quat',
'robot0_gripper_qpos',
'object'] # same order of preprocessed observations
multi_step:
n_obs_steps: ${cond_steps}
n_action_steps: ${act_steps}
max_episode_steps: ${env.max_episode_steps}
reset_within_step: True
wandb:
entity: ${oc.env:DPPO_WANDB_ENTITY}
project: calql-${env_name}
run: ${now:%H-%M-%S}_${name}
train:
n_train_itr: 1000
n_steps: 1
gamma: 0.99
actor_lr: 1e-4
actor_weight_decay: 0
actor_lr_scheduler:
first_cycle_steps: 1000
warmup_steps: 10
min_lr: 1e-4
critic_lr: 3e-4
critic_weight_decay: 0
critic_lr_scheduler:
first_cycle_steps: 1000
warmup_steps: 10
min_lr: 3e-4
save_model_freq: 10
val_freq: 10
render:
freq: 1
num: 0
log_freq: 1
# CalQL specific
train_online: False
batch_size: 256
n_random_actions: 4
target_ema_rate: 0.005
scale_reward_factor: 1.0
num_update: 1000
buffer_size: 1000000
n_eval_episode: 10
n_explore_steps: 0
target_entropy: ${eval:'- ${action_dim} * ${act_steps}'}
init_temperature: 1
automatic_entropy_tuning: True
model:
_target_: model.rl.gaussian_calql.CalQL_Gaussian
randn_clip_value: 3
cql_min_q_weight: 5.0
tanh_output: True
actor:
_target_: model.common.mlp_gaussian.Gaussian_MLP
mlp_dims: [512, 512, 512]
activation_type: ReLU
tanh_output: False # squash after sampling instead
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
horizon_steps: ${horizon_steps}
action_dim: ${action_dim}
std_max: 7.3891
std_min: 0.0067
critic:
_target_: model.common.critic.CriticObsAct
mlp_dims: [256, 256, 256]
activation_type: ReLU
use_layernorm: True
double_q: True
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
action_dim: ${action_dim}
action_steps: ${act_steps}
horizon_steps: ${horizon_steps}
device: ${device}
offline_dataset:
_target_: agent.dataset.sequence.StitchedSequenceQLearningDataset
dataset_path: ${offline_dataset_path}
horizon_steps: ${horizon_steps}
cond_steps: ${cond_steps}
device: ${device}
discount_factor: ${train.gamma}
get_mc_return: True
cfg/robomimic/pretrain/can/pre_diffusion_mlp_ta1.yaml
+65
View File
@@ -0,0 +1,65 @@
defaults:
- _self_
hydra:
run:
dir: ${logdir}
_target_: agent.pretrain.train_diffusion_agent.TrainDiffusionAgent
name: ${env}_pre_diffusion_mlp_ta${horizon_steps}_td${denoising_steps}
logdir: ${oc.env:DPPO_LOG_DIR}/robomimic-pretrain/${name}/${now:%Y-%m-%d}_${now:%H-%M-%S}_${seed}
train_dataset_path: ${oc.env:DPPO_DATA_DIR}/robomimic/${env}/train.npz
seed: 42
device: cuda:0
env: can
obs_dim: 23
action_dim: 7
denoising_steps: 20
horizon_steps: 1
cond_steps: 1
wandb:
entity: ${oc.env:DPPO_WANDB_ENTITY}
project: robomimic-${env}-pretrain
run: ${now:%H-%M-%S}_${name}
train:
n_epochs: 8000
batch_size: 256
learning_rate: 1e-4
weight_decay: 1e-6
lr_scheduler:
first_cycle_steps: 10000
warmup_steps: 100
min_lr: 1e-5
epoch_start_ema: 20
update_ema_freq: 10
save_model_freq: 1000
model:
_target_: model.diffusion.diffusion.DiffusionModel
predict_epsilon: True
denoised_clip_value: 1.0
network:
_target_: model.diffusion.mlp_diffusion.DiffusionMLP
time_dim: 16
mlp_dims: [512, 512, 512]
residual_style: True
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
horizon_steps: ${horizon_steps}
action_dim: ${action_dim}
horizon_steps: ${horizon_steps}
obs_dim: ${obs_dim}
action_dim: ${action_dim}
denoising_steps: ${denoising_steps}
device: ${device}
ema:
decay: 0.995
train_dataset:
_target_: agent.dataset.sequence.StitchedSequenceDataset
dataset_path: ${train_dataset_path}
horizon_steps: ${horizon_steps}
cond_steps: ${cond_steps}
device: ${device}
cfg/robomimic/pretrain/can/pre_diffusion_mlp_ta1_ph.yaml
+65
View File
@@ -0,0 +1,65 @@
defaults:
- _self_
hydra:
run:
dir: ${logdir}
_target_: agent.pretrain.train_diffusion_agent.TrainDiffusionAgent
name: ${env}_pre_diffusion_mlp_ta${horizon_steps}_td${denoising_steps}
logdir: ${oc.env:DPPO_LOG_DIR}/robomimic-pretrain/${name}/${now:%Y-%m-%d}_${now:%H-%M-%S}_${seed}
train_dataset_path: ${oc.env:DPPO_DATA_DIR}/robomimic/${env}-ph/train.npz
seed: 42
device: cuda:0
env: can
obs_dim: 23
action_dim: 7
denoising_steps: 20
horizon_steps: 1
cond_steps: 1
wandb:
entity: ${oc.env:DPPO_WANDB_ENTITY}
project: robomimic-${env}-pretrain
run: ${now:%H-%M-%S}_${name}
train:
n_epochs: 8000
batch_size: 256
learning_rate: 1e-4
weight_decay: 1e-6
lr_scheduler:
first_cycle_steps: 10000
warmup_steps: 100
min_lr: 1e-5
epoch_start_ema: 20
update_ema_freq: 10
save_model_freq: 1000
model:
_target_: model.diffusion.diffusion.DiffusionModel
predict_epsilon: True
denoised_clip_value: 1.0
network:
_target_: model.diffusion.mlp_diffusion.DiffusionMLP
time_dim: 16
mlp_dims: [512, 512, 512]
residual_style: True
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
horizon_steps: ${horizon_steps}
action_dim: ${action_dim}
horizon_steps: ${horizon_steps}
obs_dim: ${obs_dim}
action_dim: ${action_dim}
denoising_steps: ${denoising_steps}
device: ${device}
ema:
decay: 0.995
train_dataset:
_target_: agent.dataset.sequence.StitchedSequenceDataset
dataset_path: ${train_dataset_path}
horizon_steps: ${horizon_steps}
cond_steps: ${cond_steps}
device: ${device}
cfg/robomimic/pretrain/transport/pre_gaussian_mlp_ibrl.yaml → cfg/robomimic/pretrain/can/pre_gaussian_mlp_ta1_ph.yaml
+7 -7
View File
@@ -7,13 +7,13 @@ _target_: agent.pretrain.train_gaussian_agent.TrainGaussianAgent
name: ${env}_pre_gaussian_mlp_ta${horizon_steps}
logdir: ${oc.env:DPPO_LOG_DIR}/robomimic-pretrain/${name}/${now:%Y-%m-%d}_${now:%H-%M-%S}_${seed}
train_dataset_path: ${oc.env:DPPO_DATA_DIR}/robomimic/${env}/train.npz
train_dataset_path: ${oc.env:DPPO_DATA_DIR}/robomimic/${env}-ph/train.npz
seed: 42
device: cuda:0
env: transport
obs_dim: 59
action_dim: 14
env: can
obs_dim: 23
action_dim: 7
horizon_steps: 1
cond_steps: 1
@@ -26,11 +26,11 @@ train:
n_epochs: 5000
batch_size: 256
learning_rate: 1e-4
weight_decay: 0
weight_decay: 1e-6
lr_scheduler:
first_cycle_steps: 5000
warmup_steps: 100
min_lr: 1e-4
min_lr: 1e-5
epoch_start_ema: 20
update_ema_freq: 10
save_model_freq: 1000
@@ -45,7 +45,7 @@ model:
fixed_std: 0.1
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
horizon_steps: ${horizon_steps}
action_dim: ${action_dim}
action_dim: ${action_dim}
horizon_steps: ${horizon_steps}
device: ${device}
cfg/robomimic/pretrain/square/calql_mlp_offline.yaml
+5 -5
View File
@@ -46,7 +46,7 @@ wandb:
run: ${now:%H-%M-%S}_${name}
train:
n_train_itr: 100
n_train_itr: 1000
n_steps: 1
gamma: 0.99
actor_lr: 1e-4
@@ -61,8 +61,8 @@ train:
first_cycle_steps: 1000
warmup_steps: 10
min_lr: 3e-4
save_model_freq: 10
val_freq: 10
save_model_freq: 100
val_freq: 20
render:
freq: 1
num: 0
@@ -70,7 +70,7 @@ train:
# CalQL specific
train_online: False
batch_size: 256
n_random_actions: 4
n_random_actions: 10
target_ema_rate: 0.005
scale_reward_factor: 1.0
num_update: 1000
@@ -93,7 +93,7 @@ model:
tanh_output: False # squash after sampling instead
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
horizon_steps: ${horizon_steps}
action_dim: ${action_dim}
std_max: 7.3891
std_min: 0.0067
critic:
cfg/robomimic/pretrain/square/calql_mlp_offline_ph.yaml
+118
View File
@@ -0,0 +1,118 @@
defaults:
- _self_
hydra:
run:
dir: ${logdir}
_target_: agent.finetune.train_calql_agent.TrainCalQLAgent
name: ${env_name}_calql_mlp_ta${horizon_steps}
logdir: ${oc.env:DPPO_LOG_DIR}/robomimic-pretrain/${name}/${now:%Y-%m-%d}_${now:%H-%M-%S}_${seed}
robomimic_env_cfg_path: cfg/robomimic/env_meta/${env_name}.json
normalization_path: ${oc.env:DPPO_DATA_DIR}/robomimic/${env_name}-ph/normalization.npz
offline_dataset_path: ${oc.env:DPPO_DATA_DIR}/robomimic/${env_name}-ph/train.npz
seed: 42
device: cuda:0
env_name: square
obs_dim: 23
action_dim: 7
cond_steps: 1
horizon_steps: 1
act_steps: 1
env:
n_envs: 1
name: ${env_name}
best_reward_threshold_for_success: 1
max_episode_steps: 400
reset_at_iteration: False
save_video: False
wrappers:
robomimic_lowdim:
normalization_path: ${normalization_path}
low_dim_keys: ['robot0_eef_pos',
'robot0_eef_quat',
'robot0_gripper_qpos',
'object'] # same order of preprocessed observations
multi_step:
n_obs_steps: ${cond_steps}
n_action_steps: ${act_steps}
max_episode_steps: ${env.max_episode_steps}
reset_within_step: True
wandb:
entity: ${oc.env:DPPO_WANDB_ENTITY}
project: calql-${env_name}
run: ${now:%H-%M-%S}_${name}
train:
n_train_itr: 1000
n_steps: 1
gamma: 0.99
actor_lr: 1e-4
actor_weight_decay: 0
actor_lr_scheduler:
first_cycle_steps: 1000
warmup_steps: 10
min_lr: 1e-4
critic_lr: 3e-4
critic_weight_decay: 0
critic_lr_scheduler:
first_cycle_steps: 1000
warmup_steps: 10
min_lr: 3e-4
save_model_freq: 10
val_freq: 10
render:
freq: 1
num: 0
log_freq: 1
# CalQL specific
train_online: False
batch_size: 256
n_random_actions: 4
target_ema_rate: 0.005
scale_reward_factor: 1.0
num_update: 1000
buffer_size: 1000000
n_eval_episode: 10
n_explore_steps: 0
target_entropy: ${eval:'- ${action_dim} * ${act_steps}'}
init_temperature: 1
automatic_entropy_tuning: True
model:
_target_: model.rl.gaussian_calql.CalQL_Gaussian
randn_clip_value: 3
cql_min_q_weight: 5.0
tanh_output: True
actor:
_target_: model.common.mlp_gaussian.Gaussian_MLP
mlp_dims: [512, 512, 512]
activation_type: ReLU
tanh_output: False # squash after sampling instead
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
horizon_steps: ${horizon_steps}
action_dim: ${action_dim}
std_max: 7.3891
std_min: 0.0067
critic:
_target_: model.common.critic.CriticObsAct
mlp_dims: [256, 256, 256]
activation_type: ReLU
use_layernorm: True
double_q: True
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
action_dim: ${action_dim}
action_steps: ${act_steps}
horizon_steps: ${horizon_steps}
device: ${device}
offline_dataset:
_target_: agent.dataset.sequence.StitchedSequenceQLearningDataset
dataset_path: ${offline_dataset_path}
horizon_steps: ${horizon_steps}
cond_steps: ${cond_steps}
device: ${device}
discount_factor: ${train.gamma}
get_mc_return: True
cfg/robomimic/pretrain/square/pre_diffusion_mlp_ta1.yaml
+66
View File
@@ -0,0 +1,66 @@
defaults:
- _self_
hydra:
run:
dir: ${logdir}
_target_: agent.pretrain.train_diffusion_agent.TrainDiffusionAgent
name: ${env}_pre_diffusion_mlp_ta${horizon_steps}_td${denoising_steps}
logdir: ${oc.env:DPPO_LOG_DIR}/robomimic-pretrain/${name}/${now:%Y-%m-%d}_${now:%H-%M-%S}_${seed}
train_dataset_path: ${oc.env:DPPO_DATA_DIR}/robomimic/${env}/train.npz
seed: 42
device: cuda:0
env: square
obs_dim: 23
action_dim: 7
denoising_steps: 20
horizon_steps: 1
cond_steps: 1
wandb:
entity: ${oc.env:DPPO_WANDB_ENTITY}
project: robomimic-${env}-pretrain
run: ${now:%H-%M-%S}_${name}
train:
n_epochs: 8000
batch_size: 256
learning_rate: 1e-4
weight_decay: 1e-6
lr_scheduler:
first_cycle_steps: 10000
warmup_steps: 100
min_lr: 1e-5
epoch_start_ema: 20
update_ema_freq: 10
save_model_freq: 1000
model:
_target_: model.diffusion.diffusion.DiffusionModel
predict_epsilon: True
denoised_clip_value: 1.0
network:
_target_: model.diffusion.mlp_diffusion.DiffusionMLP
time_dim: 32
mlp_dims: [1024, 1024, 1024]
cond_mlp_dims: [512, 64]
residual_style: True
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
horizon_steps: ${horizon_steps}
action_dim: ${action_dim}
horizon_steps: ${horizon_steps}
obs_dim: ${obs_dim}
action_dim: ${action_dim}
denoising_steps: ${denoising_steps}
device: ${device}
ema:
decay: 0.995
train_dataset:
_target_: agent.dataset.sequence.StitchedSequenceDataset
dataset_path: ${train_dataset_path}
horizon_steps: ${horizon_steps}
cond_steps: ${cond_steps}
device: ${device}
cfg/robomimic/pretrain/square/pre_diffusion_mlp_ta1_ph.yaml
+66
View File
@@ -0,0 +1,66 @@
defaults:
- _self_
hydra:
run:
dir: ${logdir}
_target_: agent.pretrain.train_diffusion_agent.TrainDiffusionAgent
name: ${env}_pre_diffusion_mlp_ta${horizon_steps}_td${denoising_steps}
logdir: ${oc.env:DPPO_LOG_DIR}/robomimic-pretrain/${name}/${now:%Y-%m-%d}_${now:%H-%M-%S}_${seed}
train_dataset_path: ${oc.env:DPPO_DATA_DIR}/robomimic/${env}-ph/train.npz
seed: 42
device: cuda:0
env: square
obs_dim: 23
action_dim: 7
denoising_steps: 20
horizon_steps: 1
cond_steps: 1
wandb:
entity: ${oc.env:DPPO_WANDB_ENTITY}
project: robomimic-${env}-pretrain
run: ${now:%H-%M-%S}_${name}
train:
n_epochs: 8000
batch_size: 256
learning_rate: 1e-4
weight_decay: 1e-6
lr_scheduler:
first_cycle_steps: 10000
warmup_steps: 100
min_lr: 1e-5
epoch_start_ema: 20
update_ema_freq: 10
save_model_freq: 1000
model:
_target_: model.diffusion.diffusion.DiffusionModel
predict_epsilon: True
denoised_clip_value: 1.0
network:
_target_: model.diffusion.mlp_diffusion.DiffusionMLP
time_dim: 32
mlp_dims: [1024, 1024, 1024]
cond_mlp_dims: [512, 64]
residual_style: True
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
horizon_steps: ${horizon_steps}
action_dim: ${action_dim}
horizon_steps: ${horizon_steps}
obs_dim: ${obs_dim}
action_dim: ${action_dim}
denoising_steps: ${denoising_steps}
device: ${device}
ema:
decay: 0.995
train_dataset:
_target_: agent.dataset.sequence.StitchedSequenceDataset
dataset_path: ${train_dataset_path}
horizon_steps: ${horizon_steps}
cond_steps: ${cond_steps}
device: ${device}
cfg/robomimic/pretrain/lift/pre_gaussian_mlp_ibrl.yaml → cfg/robomimic/pretrain/square/pre_gaussian_mlp_ta1_ph.yaml
+7 -6
View File
@@ -7,12 +7,12 @@ _target_: agent.pretrain.train_gaussian_agent.TrainGaussianAgent
name: ${env}_pre_gaussian_mlp_ta${horizon_steps}
logdir: ${oc.env:DPPO_LOG_DIR}/robomimic-pretrain/${name}/${now:%Y-%m-%d}_${now:%H-%M-%S}_${seed}
train_dataset_path: ${oc.env:DPPO_DATA_DIR}/robomimic/${env}/train.npz
train_dataset_path: ${oc.env:DPPO_DATA_DIR}/robomimic/${env}-ph/train.npz
seed: 42
device: cuda:0
env: lift
obs_dim: 19
env: square
obs_dim: 23
action_dim: 7
horizon_steps: 1
cond_steps: 1
@@ -40,14 +40,15 @@ model:
network:
_target_: model.common.mlp_gaussian.Gaussian_MLP
mlp_dims: [1024, 1024, 1024]
residual_style: False
activation_type: ReLU
dropout: 0.5
fixed_std: 0.1
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
horizon_steps: ${horizon_steps}
action_dim: ${action_dim}
action_dim: ${action_dim}
horizon_steps: ${horizon_steps}
device: ${device}
ema:
decay: 0.995