dodox/dppo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

v0.6 (#18)

Browse Source 
* Sampling over both env and denoising steps in DPPO updates (#13)

* sample one from each chain

* full random sampling

* Add Proficient Human (PH) Configs and Pipeline (#16)

* fix missing cfg

* add ph config

* fix how terminated flags are added to buffer in ibrl

* add ph config

* offline calql for 1M gradient updates

* bug fix: number of calql online gradient steps is the number of new transitions collected

* add sample config for DPPO with ta=1

* Sampling over both env and denoising steps in DPPO updates (#13)

* sample one from each chain

* full random sampling

* fix diffusion loss when predicting initial noise

* fix dppo inds

* fix typo

* remove print statement

---------

Co-authored-by: Justin M. Lidard <jlidard@neuronic.cs.princeton.edu>
Co-authored-by: allenzren <allen.ren@princeton.edu>

* update robomimic configs

* better calql formulation

* optimize calql and ibrl training

* optimize data transfer in ppo agents

* add kitchen configs

* re-organize config folders, rerun calql and rlpd

* add scratch gym locomotion configs

* add kitchen installation dependencies

* use truncated for termination in furniture env

* update furniture and gym configs

* update README and dependencies with kitchen

* add url for new data and checkpoints

* update demo RL configs

* update batch sizes for furniture unet configs

* raise error about dropout in residual mlp

* fix observation bug in bc loss

---------

Co-authored-by: Justin Lidard <60638575+jlidard@users.noreply.github.com>
Co-authored-by: Justin M. Lidard <jlidard@neuronic.cs.princeton.edu>
This commit is contained in:
Allen Z. Ren
2024-10-30 19:58:06 -04:00
committed by GitHub
co-authored by Justin M. Lidard Justin Lidard
parent 7b10df690d
commit dc8e0c9edc
126 changed files with 4614 additions and 553 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cfg/robomimic/finetune/can/calql_mlp_online.yaml
+2 -2
View File
@@ -7,7 +7,7 @@ _target_: agent.finetune.train_calql_agent.TrainCalQLAgent
name: ${env_name}_calql_mlp_ta${horizon_steps}
logdir: ${oc.env:DPPO_LOG_DIR}/robomimic-finetune/${name}/${now:%Y-%m-%d}_${now:%H-%M-%S}_${seed}
base_policy_path:
base_policy_path: ${oc.env:DPPO_LOG_DIR}/robomimic-pretrain/can_calql_mlp_ta1/2024-10-25_22-30-16_42/checkpoint/state_999.pt
robomimic_env_cfg_path: cfg/robomimic/env_meta/${env_name}.json
normalization_path: ${oc.env:DPPO_DATA_DIR}/robomimic/${env_name}/normalization.npz
offline_dataset_path: ${oc.env:DPPO_DATA_DIR}/robomimic/${env_name}/train.npz
@@ -97,7 +97,7 @@ model:
tanh_output: False # squash after sampling instead
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
horizon_steps: ${horizon_steps}
action_dim: ${action_dim}
std_max: 7.3891
std_min: 0.0067
critic:
cfg/robomimic/finetune/can/calql_mlp_online_ph.yaml
+122
View File
@@ -0,0 +1,122 @@
defaults:
- _self_
hydra:
run:
dir: ${logdir}
_target_: agent.finetune.train_calql_agent.TrainCalQLAgent
name: ${env_name}_calql_mlp_ta${horizon_steps}
logdir: ${oc.env:DPPO_LOG_DIR}/robomimic-finetune/${name}/${now:%Y-%m-%d}_${now:%H-%M-%S}_${seed}
base_policy_path: ${oc.env:DPPO_LOG_DIR}/robomimic-pretrain/can_calql_mlp_ta1/2024-10-09_11-05-07_0/checkpoint/state_999.pt
robomimic_env_cfg_path: cfg/robomimic/env_meta/${env_name}.json
normalization_path: ${oc.env:DPPO_DATA_DIR}/robomimic/${env_name}-ph/normalization.npz
offline_dataset_path: ${oc.env:DPPO_DATA_DIR}/robomimic/${env_name}-ph/train.npz
seed: 42
device: cuda:0
env_name: can
obs_dim: 23
action_dim: 7
cond_steps: 1
horizon_steps: 1
act_steps: 1
env:
n_envs: 1
name: ${env_name}
best_reward_threshold_for_success: 1
max_episode_steps: 300
reset_at_iteration: False
save_video: False
wrappers:
robomimic_lowdim:
normalization_path: ${normalization_path}
low_dim_keys: ['robot0_eef_pos',
'robot0_eef_quat',
'robot0_gripper_qpos',
'object'] # same order of preprocessed observations
multi_step:
n_obs_steps: ${cond_steps}
n_action_steps: ${act_steps}
max_episode_steps: ${env.max_episode_steps}
reset_within_step: True
wandb:
entity: ${oc.env:DPPO_WANDB_ENTITY}
project: calql-${env_name}
run: ${now:%H-%M-%S}_${name}
train:
n_train_itr: 1000
n_steps: 1 # not used
n_episode_per_epoch: 1
gamma: 0.99
actor_lr: 1e-4
actor_weight_decay: 0
actor_lr_scheduler:
first_cycle_steps: 1000
warmup_steps: 10
min_lr: 1e-4
critic_lr: 3e-4
critic_weight_decay: 0
critic_lr_scheduler:
first_cycle_steps: 1000
warmup_steps: 10
min_lr: 3e-4
save_model_freq: 100
val_freq: 10
render:
freq: 1
num: 0
log_freq: 1
# CalQL specific
train_online: True
batch_size: 256
n_random_actions: 4
target_ema_rate: 0.005
scale_reward_factor: 1.0
num_update: 1000
buffer_size: 1000000
online_utd_ratio: 1
n_eval_episode: 40
n_explore_steps: 0
target_entropy: ${eval:'- ${action_dim} * ${act_steps}'}
init_temperature: 1
automatic_entropy_tuning: True
model:
_target_: model.rl.gaussian_calql.CalQL_Gaussian
randn_clip_value: 3
cql_min_q_weight: 5.0
tanh_output: True
network_path: ${base_policy_path}
actor:
_target_: model.common.mlp_gaussian.Gaussian_MLP
mlp_dims: [512, 512, 512]
activation_type: ReLU
tanh_output: False # squash after sampling instead
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
horizon_steps: ${horizon_steps}
action_dim: ${action_dim}
std_max: 7.3891
std_min: 0.0067
critic:
_target_: model.common.critic.CriticObsAct
mlp_dims: [256, 256, 256]
activation_type: ReLU
use_layernorm: True
double_q: True
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
action_dim: ${action_dim}
action_steps: ${act_steps}
horizon_steps: ${horizon_steps}
device: ${device}
offline_dataset:
_target_: agent.dataset.sequence.StitchedSequenceQLearningDataset
dataset_path: ${offline_dataset_path}
horizon_steps: ${horizon_steps}
cond_steps: ${cond_steps}
device: ${device}
discount_factor: ${train.gamma}
get_mc_return: True
cfg/robomimic/finetune/can/ft_awr_diffusion_mlp.yaml
+2 -2
View File
@@ -26,7 +26,7 @@ env:
name: ${env_name}
best_reward_threshold_for_success: 1
max_episode_steps: 300
save_video: false
save_video: False
wrappers:
robomimic_lowdim:
normalization_path: ${normalization_path}
@@ -46,7 +46,7 @@ wandb:
run: ${now:%H-%M-%S}_${name}
train:
n_train_itr: 300
n_train_itr: 151
n_critic_warmup_itr: 2
n_steps: 300
gamma: 0.999
cfg/robomimic/finetune/can/ft_dql_diffusion_mlp.yaml
+1 -1
View File
@@ -26,7 +26,7 @@ env:
name: ${env_name}
best_reward_threshold_for_success: 1
max_episode_steps: 300
save_video: false
save_video: False
wrappers:
robomimic_lowdim:
normalization_path: ${normalization_path}
cfg/robomimic/finetune/can/ft_idql_diffusion_mlp.yaml
+1 -1
View File
@@ -46,7 +46,7 @@ wandb:
run: ${now:%H-%M-%S}_${name}
train:
n_train_itr: 300
n_train_itr: 151
n_critic_warmup_itr: 5
n_steps: 300
gamma: 0.999
cfg/robomimic/finetune/can/ft_ppo_diffusion_mlp.yaml
+3 -3
View File
@@ -47,16 +47,16 @@ wandb:
run: ${now:%H-%M-%S}_${name}
train:
n_train_itr: 300
n_train_itr: 151
n_critic_warmup_itr: 2
n_steps: 300
gamma: 0.999
actor_lr: 1e-5
actor_lr: 1e-4
actor_weight_decay: 0
actor_lr_scheduler:
first_cycle_steps: 1000
warmup_steps: 10
min_lr: 1e-5
min_lr: 1e-4
critic_lr: 1e-3
critic_weight_decay: 0
critic_lr_scheduler:
cfg/robomimic/finetune/can/ft_ppo_diffusion_mlp_img.yaml
+8 -8
View File
@@ -1,7 +1,7 @@
defaults:
- _self_
hydra:
run:
run:
dir: ${logdir}
_target_: agent.finetune.train_ppo_diffusion_img_agent.TrainPPOImgDiffusionAgent
@@ -60,22 +60,22 @@ wandb:
run: ${now:%H-%M-%S}_${name}
train:
n_train_itr: 200
n_train_itr: 151
n_critic_warmup_itr: 2
n_steps: 300
gamma: 0.999
augment: True
grad_accumulate: 15
actor_lr: 1e-5
actor_lr: 1e-4
actor_weight_decay: 0
actor_lr_scheduler:
first_cycle_steps: 200
first_cycle_steps: 1000
warmup_steps: 10
min_lr: 1e-5
min_lr: 1e-4
critic_lr: 1e-3
critic_weight_decay: 0
critic_lr_scheduler:
first_cycle_steps: 200
first_cycle_steps: 1000
warmup_steps: 10
min_lr: 1e-3
save_model_freq: 100
@@ -96,7 +96,7 @@ train:
model:
_target_: model.diffusion.diffusion_ppo.PPODiffusion
# HP to tune
gamma_denoising: 0.9
gamma_denoising: 0.99
clip_ploss_coef: 0.01
clip_ploss_coef_base: 0.001
clip_ploss_coef_rate: 3
@@ -158,10 +158,10 @@ model:
embed_style: embed2
embed_norm: 0
img_cond_steps: ${img_cond_steps}
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
mlp_dims: [256, 256, 256]
activation_type: Mish
residual_style: True
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
ft_denoising_steps: ${ft_denoising_steps}
horizon_steps: ${horizon_steps}
obs_dim: ${obs_dim}
cfg/robomimic/finetune/can/ft_ppo_diffusion_mlp_ta1.yaml
+111
View File
@@ -0,0 +1,111 @@
defaults:
- _self_
hydra:
run:
dir: ${logdir}
_target_: agent.finetune.train_ppo_diffusion_agent.TrainPPODiffusionAgent
name: ${env_name}_ft_diffusion_mlp_ta${horizon_steps}_td${denoising_steps}_tdf${ft_denoising_steps}
logdir: ${oc.env:DPPO_LOG_DIR}/robomimic-finetune/${name}/${now:%Y-%m-%d}_${now:%H-%M-%S}_${seed}
base_policy_path: ${oc.env:DPPO_LOG_DIR}/robomimic-pretrain/can_pre_diffusion_mlp_ta1_td20/2024-09-29_15-43-07_42/checkpoint/state_8000.pt
robomimic_env_cfg_path: cfg/robomimic/env_meta/${env_name}.json
normalization_path: ${oc.env:DPPO_DATA_DIR}/robomimic/${env_name}/normalization.npz
seed: 42
device: cuda:0
env_name: can
obs_dim: 23
action_dim: 7
denoising_steps: 20
ft_denoising_steps: 10
cond_steps: 1
horizon_steps: 1
act_steps: 1
env:
n_envs: 50
name: ${env_name}
best_reward_threshold_for_success: 1
max_episode_steps: 300
save_video: False
wrappers:
robomimic_lowdim:
normalization_path: ${normalization_path}
low_dim_keys: ['robot0_eef_pos',
'robot0_eef_quat',
'robot0_gripper_qpos',
'object'] # same order of preprocessed observations
multi_step:
n_obs_steps: ${cond_steps}
n_action_steps: ${act_steps}
max_episode_steps: ${env.max_episode_steps}
reset_within_step: True
wandb:
entity: ${oc.env:DPPO_WANDB_ENTITY}
project: robomimic-${env_name}-finetune
run: ${now:%H-%M-%S}_${name}
train:
n_train_itr: 301
n_critic_warmup_itr: 2
n_steps: 300
gamma: 0.999
actor_lr: 1e-4
actor_weight_decay: 0
actor_lr_scheduler:
first_cycle_steps: 1000
warmup_steps: 10
min_lr: 1e-4
critic_lr: 1e-3
critic_weight_decay: 0
critic_lr_scheduler:
first_cycle_steps: 1000
warmup_steps: 10
min_lr: 1e-3
save_model_freq: 100
val_freq: 10
render:
freq: 1
num: 0
# PPO specific
reward_scale_running: True
reward_scale_const: 1.0
gae_lambda: 0.95
batch_size: 15000
update_epochs: 10
vf_coef: 0.5
target_kl: 1
model:
_target_: model.diffusion.diffusion_ppo.PPODiffusion
# HP to tune
gamma_denoising: 0.99
clip_ploss_coef: 0.01
clip_ploss_coef_base: 0.001
clip_ploss_coef_rate: 3
randn_clip_value: 3
min_sampling_denoising_std: 0.1
min_logprob_denoising_std: 0.1
#
network_path: ${base_policy_path}
actor:
_target_: model.diffusion.mlp_diffusion.DiffusionMLP
time_dim: 16
mlp_dims: [512, 512, 512]
residual_style: True
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
horizon_steps: ${horizon_steps}
action_dim: ${action_dim}
critic:
_target_: model.common.critic.CriticObs
mlp_dims: [256, 256, 256]
activation_type: Mish
residual_style: True
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
ft_denoising_steps: ${ft_denoising_steps}
horizon_steps: ${horizon_steps}
obs_dim: ${obs_dim}
action_dim: ${action_dim}
denoising_steps: ${denoising_steps}
device: ${device}
cfg/robomimic/finetune/can/ft_ppo_diffusion_mlp_ta1_ph.yaml
+111
View File
@@ -0,0 +1,111 @@
defaults:
- _self_
hydra:
run:
dir: ${logdir}
_target_: agent.finetune.train_ppo_diffusion_agent.TrainPPODiffusionAgent
name: ${env_name}_ft_diffusion_mlp_ta${horizon_steps}_td${denoising_steps}_tdf${ft_denoising_steps}
logdir: ${oc.env:DPPO_LOG_DIR}/robomimic-finetune/${name}/${now:%Y-%m-%d}_${now:%H-%M-%S}_${seed}
base_policy_path: ${oc.env:DPPO_LOG_DIR}/robomimic-pretrain/can_pre_diffusion_mlp_ta1_td20/2024-10-14_10-54-33_0/checkpoint/state_5000.pt
robomimic_env_cfg_path: cfg/robomimic/env_meta/${env_name}.json
normalization_path: ${oc.env:DPPO_DATA_DIR}/robomimic/${env_name}-ph/normalization.npz
seed: 42
device: cuda:0
env_name: can
obs_dim: 23
action_dim: 7
denoising_steps: 20
ft_denoising_steps: 10
cond_steps: 1
horizon_steps: 1
act_steps: 1
env:
n_envs: 40
name: ${env_name}
best_reward_threshold_for_success: 1
max_episode_steps: 300
save_video: False
wrappers:
robomimic_lowdim:
normalization_path: ${normalization_path}
low_dim_keys: ['robot0_eef_pos',
'robot0_eef_quat',
'robot0_gripper_qpos',
'object'] # same order of preprocessed observations
multi_step:
n_obs_steps: ${cond_steps}
n_action_steps: ${act_steps}
max_episode_steps: ${env.max_episode_steps}
reset_within_step: True
wandb:
entity: ${oc.env:DPPO_WANDB_ENTITY}
project: robomimic-${env_name}-finetune
run: ${now:%H-%M-%S}_${name}
train:
n_train_itr: 301
n_critic_warmup_itr: 2
n_steps: 300
gamma: 0.999
actor_lr: 1e-4
actor_weight_decay: 0
actor_lr_scheduler:
first_cycle_steps: 1000
warmup_steps: 10
min_lr: 1e-4
critic_lr: 1e-3
critic_weight_decay: 0
critic_lr_scheduler:
first_cycle_steps: 1000
warmup_steps: 10
min_lr: 1e-3
save_model_freq: 100
val_freq: 10
render:
freq: 1
num: 0
# PPO specific
reward_scale_running: True
reward_scale_const: 1.0
gae_lambda: 0.95
batch_size: 6000
update_epochs: 10
vf_coef: 0.5
target_kl: 1
model:
_target_: model.diffusion.diffusion_ppo.PPODiffusion
# HP to tune
gamma_denoising: 0.9
clip_ploss_coef: 0.01
clip_ploss_coef_base: 0.001
clip_ploss_coef_rate: 3
randn_clip_value: 3
min_sampling_denoising_std: 0.1
min_logprob_denoising_std: 0.1
#
network_path: ${base_policy_path}
actor:
_target_: model.diffusion.mlp_diffusion.DiffusionMLP
time_dim: 16
mlp_dims: [512, 512, 512]
residual_style: True
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
horizon_steps: ${horizon_steps}
action_dim: ${action_dim}
critic:
_target_: model.common.critic.CriticObs
mlp_dims: [256, 256, 256]
activation_type: Mish
residual_style: True
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
ft_denoising_steps: ${ft_denoising_steps}
horizon_steps: ${horizon_steps}
obs_dim: ${obs_dim}
action_dim: ${action_dim}
denoising_steps: ${denoising_steps}
device: ${device}
cfg/robomimic/finetune/can/ft_qsm_diffusion_mlp.yaml
+1 -1
View File
@@ -46,7 +46,7 @@ wandb:
run: ${now:%H-%M-%S}_${name}
train:
n_train_itr: 300
n_train_itr: 151
n_critic_warmup_itr: 5
n_steps: 300
gamma: 0.999
cfg/robomimic/finetune/can/ft_rwr_diffusion_mlp.yaml
+2 -2
View File
@@ -26,7 +26,7 @@ env:
name: ${env_name}
best_reward_threshold_for_success: 1
max_episode_steps: 300
save_video: false
save_video: False
wrappers:
robomimic_lowdim:
normalization_path: ${normalization_path}
@@ -46,7 +46,7 @@ wandb:
run: ${now:%H-%M-%S}_${name}
train:
n_train_itr: 300
n_train_itr: 151
n_critic_warmup_itr: 2
n_steps: 300
gamma: 0.999
cfg/robomimic/finetune/can/ibrl_mlp.yaml
+2 -2
View File
@@ -7,7 +7,7 @@ _target_: agent.finetune.train_ibrl_agent.TrainIBRLAgent
name: ${env_name}_ibrl_mlp_ta${horizon_steps}
logdir: ${oc.env:DPPO_LOG_DIR}/robomimic-finetune/${name}/${now:%Y-%m-%d}_${now:%H-%M-%S}_${seed}
base_policy_path:
base_policy_path: ${oc.env:DPPO_LOG_DIR}/robomimic-pretrain/can_pre_gaussian_mlp_ta1/2024-09-28_13-43-59_42/checkpoint/state_5000.pt
robomimic_env_cfg_path: cfg/robomimic/env_meta/${env_name}.json
normalization_path: ${oc.env:DPPO_DATA_DIR}/robomimic/${env_name}/normalization.npz
offline_dataset_path: ${oc.env:DPPO_DATA_DIR}/robomimic/${env_name}/train.npz
@@ -93,7 +93,7 @@ model:
fixed_std: 0.1
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
horizon_steps: ${horizon_steps}
action_dim: ${action_dim}
critic:
_target_: model.common.critic.CriticObsAct
mlp_dims: [1024, 1024, 1024]
cfg/robomimic/finetune/can/ibrl_mlp_ph.yaml
+115
View File
@@ -0,0 +1,115 @@
defaults:
- _self_
hydra:
run:
dir: ${logdir}
_target_: agent.finetune.train_ibrl_agent.TrainIBRLAgent
name: ${env_name}_ibrl_mlp_ta${horizon_steps}
logdir: ${oc.env:DPPO_LOG_DIR}/robomimic-finetune/${name}/${now:%Y-%m-%d}_${now:%H-%M-%S}_${seed}
base_policy_path: ${oc.env:DPPO_LOG_DIR}/robomimic-pretrain/can_pre_gaussian_mlp_ta1/2024-10-08_20-52-04_0/checkpoint/state_5000.pt
robomimic_env_cfg_path: cfg/robomimic/env_meta/${env_name}.json
normalization_path: ${oc.env:DPPO_DATA_DIR}/robomimic/${env_name}-ph/normalization.npz
offline_dataset_path: ${oc.env:DPPO_DATA_DIR}/robomimic/${env_name}-ph/train.npz
seed: 42
device: cuda:0
env_name: can
obs_dim: 23
action_dim: 7
cond_steps: 1
horizon_steps: 1
act_steps: 1
env:
n_envs: 1
name: ${env_name}
max_episode_steps: 250 # IBRL uses 200
reset_at_iteration: False
save_video: False
best_reward_threshold_for_success: 1
wrappers:
robomimic_lowdim:
normalization_path: ${normalization_path}
low_dim_keys: ['robot0_eef_pos',
'robot0_eef_quat',
'robot0_gripper_qpos',
'object'] # same order of preprocessed observations
multi_step:
n_obs_steps: ${cond_steps}
n_action_steps: ${act_steps}
max_episode_steps: ${env.max_episode_steps}
reset_within_step: True
wandb:
entity: ${oc.env:DPPO_WANDB_ENTITY}
project: ibrl-${env_name}
run: ${now:%H-%M-%S}_${name}
train:
n_train_itr: 1000000
n_steps: 1
gamma: 0.99
actor_lr: 1e-4
actor_weight_decay: 0
actor_lr_scheduler:
first_cycle_steps: 1000
warmup_steps: 10
min_lr: 1e-4
critic_lr: 1e-4
critic_weight_decay: 0
critic_lr_scheduler:
first_cycle_steps: 1000
warmup_steps: 10
min_lr: 1e-4
save_model_freq: 100000
val_freq: 10000
render:
freq: 10000
num: 0
log_freq: 200
# IBRL specific
batch_size: 256
target_ema_rate: 0.01
scale_reward_factor: 1
critic_num_update: 3
buffer_size: 400000
n_eval_episode: 40
n_explore_steps: 0
update_freq: 2
model:
_target_: model.rl.gaussian_ibrl.IBRL_Gaussian
randn_clip_value: 3
n_critics: 5
soft_action_sample: True
soft_action_sample_beta: 10
network_path: ${base_policy_path}
actor:
_target_: model.common.mlp_gaussian.Gaussian_MLP
mlp_dims: [1024, 1024, 1024]
activation_type: ReLU
dropout: 0.5
fixed_std: 0.1
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
horizon_steps: ${horizon_steps}
action_dim: ${action_dim}
critic:
_target_: model.common.critic.CriticObsAct
mlp_dims: [1024, 1024, 1024]
activation_type: ReLU
use_layernorm: True
double_q: False # use ensemble
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
action_dim: ${action_dim}
action_steps: ${act_steps}
horizon_steps: ${horizon_steps}
device: ${device}
offline_dataset:
_target_: agent.dataset.sequence.StitchedSequenceQLearningDataset
dataset_path: ${offline_dataset_path}
horizon_steps: ${horizon_steps}
cond_steps: ${cond_steps}
device: ${device}
max_n_episodes: 100
cfg/robomimic/finetune/lift/ft_awr_diffusion_mlp.yaml
+1 -1
View File
@@ -46,7 +46,7 @@ wandb:
run: ${now:%H-%M-%S}_${name}
train:
n_train_itr: 300
n_train_itr: 81
n_critic_warmup_itr: 2
n_steps: 300
gamma: 0.999
cfg/robomimic/finetune/lift/ft_dql_diffusion_mlp.yaml
+1 -1
View File
@@ -46,7 +46,7 @@ wandb:
run: ${now:%H-%M-%S}_${name}
train:
n_train_itr: 300
n_train_itr: 81
n_critic_warmup_itr: 5
n_steps: 300
gamma: 0.999
cfg/robomimic/finetune/lift/ft_idql_diffusion_mlp.yaml
+1 -1
View File
@@ -46,7 +46,7 @@ wandb:
run: ${now:%H-%M-%S}_${name}
train:
n_train_itr: 300
n_train_itr: 81
n_critic_warmup_itr: 5
n_steps: 300
gamma: 0.999
cfg/robomimic/finetune/lift/ft_ppo_diffusion_mlp.yaml
+5 -5
View File
@@ -27,7 +27,7 @@ env:
name: ${env_name}
best_reward_threshold_for_success: 1
max_episode_steps: 300
save_video: false
save_video: False
wrappers:
robomimic_lowdim:
normalization_path: ${normalization_path}
@@ -47,16 +47,16 @@ wandb:
run: ${now:%H-%M-%S}_${name}
train:
n_train_itr: 300
n_train_itr: 81
n_critic_warmup_itr: 2
n_steps: 300
gamma: 0.999
actor_lr: 1e-5
actor_lr: 1e-4
actor_weight_decay: 0
actor_lr_scheduler:
first_cycle_steps: 1000
warmup_steps: 10
min_lr: 1e-5
min_lr: 1e-4
critic_lr: 1e-3
critic_weight_decay: 0
critic_lr_scheduler:
@@ -99,10 +99,10 @@ model:
action_dim: ${action_dim}
critic:
_target_: model.common.critic.CriticObs
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
mlp_dims: [256, 256, 256]
activation_type: Mish
residual_style: True
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
ft_denoising_steps: ${ft_denoising_steps}
horizon_steps: ${horizon_steps}
obs_dim: ${obs_dim}
cfg/robomimic/finetune/lift/ft_ppo_diffusion_mlp_img.yaml
+8 -8
View File
@@ -1,7 +1,7 @@
defaults:
- _self_
hydra:
run:
run:
dir: ${logdir}
_target_: agent.finetune.train_ppo_diffusion_img_agent.TrainPPOImgDiffusionAgent
@@ -60,22 +60,22 @@ wandb:
run: ${now:%H-%M-%S}_${name}
train:
n_train_itr: 200
n_train_itr: 151
n_critic_warmup_itr: 2
n_steps: 300
gamma: 0.999
augment: True
grad_accumulate: 15
actor_lr: 1e-5
actor_lr: 1e-4
actor_weight_decay: 0
actor_lr_scheduler:
first_cycle_steps: 200
first_cycle_steps: 1000
warmup_steps: 10
min_lr: 1e-5
min_lr: 1e-4
critic_lr: 1e-3
critic_weight_decay: 0
critic_lr_scheduler:
first_cycle_steps: 200
first_cycle_steps: 1000
warmup_steps: 10
min_lr: 1e-3
save_model_freq: 100
@@ -96,7 +96,7 @@ train:
model:
_target_: model.diffusion.diffusion_ppo.PPODiffusion
# HP to tune
gamma_denoising: 0.9
gamma_denoising: 0.99
clip_ploss_coef: 0.01
clip_ploss_coef_base: 0.001
clip_ploss_coef_rate: 3
@@ -158,10 +158,10 @@ model:
embed_style: embed2
embed_norm: 0
img_cond_steps: ${img_cond_steps}
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
mlp_dims: [256, 256, 256]
activation_type: Mish
residual_style: True
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
ft_denoising_steps: ${ft_denoising_steps}
horizon_steps: ${horizon_steps}
obs_dim: ${obs_dim}
cfg/robomimic/finetune/lift/ft_qsm_diffusion_mlp.yaml
+1 -1
View File
@@ -46,7 +46,7 @@ wandb:
run: ${now:%H-%M-%S}_${name}
train:
n_train_itr: 300
n_train_itr: 81
n_critic_warmup_itr: 5
n_steps: 300
gamma: 0.999
cfg/robomimic/finetune/lift/ft_rwr_diffusion_mlp.yaml
+1 -1
View File
@@ -46,7 +46,7 @@ wandb:
run: ${now:%H-%M-%S}_${name}
train:
n_train_itr: 300
n_train_itr: 81
n_critic_warmup_itr: 2
n_steps: 300
gamma: 0.999
cfg/robomimic/finetune/square/calql_mlp_online.yaml
+2 -2
View File
@@ -7,7 +7,7 @@ _target_: agent.finetune.train_calql_agent.TrainCalQLAgent
name: ${env_name}_calql_mlp_ta${horizon_steps}
logdir: ${oc.env:DPPO_LOG_DIR}/robomimic-finetune/${name}/${now:%Y-%m-%d}_${now:%H-%M-%S}_${seed}
base_policy_path:
base_policy_path: ${oc.env:DPPO_LOG_DIR}/robomimic-pretrain/square_calql_mlp_ta1/2024-10-25_22-44-12_42/checkpoint/state_999.pt
robomimic_env_cfg_path: cfg/robomimic/env_meta/${env_name}.json
normalization_path: ${oc.env:DPPO_DATA_DIR}/robomimic/${env_name}/normalization.npz
offline_dataset_path: ${oc.env:DPPO_DATA_DIR}/robomimic/${env_name}/train.npz
@@ -97,7 +97,7 @@ model:
tanh_output: False # squash after sampling instead
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
horizon_steps: ${horizon_steps}
action_dim: ${action_dim}
std_max: 7.3891
std_min: 0.0067
critic:
cfg/robomimic/finetune/square/calql_mlp_online_ph.yaml
+122
View File
@@ -0,0 +1,122 @@
defaults:
- _self_
hydra:
run:
dir: ${logdir}
_target_: agent.finetune.train_calql_agent.TrainCalQLAgent
name: ${env_name}_calql_mlp_ta${horizon_steps}
logdir: ${oc.env:DPPO_LOG_DIR}/robomimic-finetune/${name}/${now:%Y-%m-%d}_${now:%H-%M-%S}_${seed}
base_policy_path: ${oc.env:DPPO_LOG_DIR}/robomimic-pretrain/square_calql_mlp_ta1/2024-10-09_11-05-07_0/checkpoint/state_999.pt
robomimic_env_cfg_path: cfg/robomimic/env_meta/${env_name}.json
normalization_path: ${oc.env:DPPO_DATA_DIR}/robomimic/${env_name}-ph/normalization.npz
offline_dataset_path: ${oc.env:DPPO_DATA_DIR}/robomimic/${env_name}-ph/train.npz
seed: 42
device: cuda:0
env_name: square
obs_dim: 23
action_dim: 7
cond_steps: 1
horizon_steps: 1
act_steps: 1
env:
n_envs: 1
name: ${env_name}
best_reward_threshold_for_success: 1
max_episode_steps: 400
reset_at_iteration: False
save_video: False
wrappers:
robomimic_lowdim:
normalization_path: ${normalization_path}
low_dim_keys: ['robot0_eef_pos',
'robot0_eef_quat',
'robot0_gripper_qpos',
'object'] # same order of preprocessed observations
multi_step:
n_obs_steps: ${cond_steps}
n_action_steps: ${act_steps}
max_episode_steps: ${env.max_episode_steps}
reset_within_step: True
wandb:
entity: ${oc.env:DPPO_WANDB_ENTITY}
project: calql-${env_name}
run: ${now:%H-%M-%S}_${name}
train:
n_train_itr: 10000
n_steps: 1 # not used
n_episode_per_epoch: 1
gamma: 0.99
actor_lr: 1e-4
actor_weight_decay: 0
actor_lr_scheduler:
first_cycle_steps: 1000
warmup_steps: 10
min_lr: 1e-4
critic_lr: 3e-4
critic_weight_decay: 0
critic_lr_scheduler:
first_cycle_steps: 1000
warmup_steps: 10
min_lr: 3e-4
save_model_freq: 100
val_freq: 10
render:
freq: 1
num: 0
log_freq: 1
# CalQL specific
train_online: True
batch_size: 256
n_random_actions: 4
target_ema_rate: 0.005
scale_reward_factor: 1.0
num_update: 1000
buffer_size: 1000000
online_utd_ratio: 1
n_eval_episode: 40
n_explore_steps: 0
target_entropy: ${eval:'- ${action_dim} * ${act_steps}'}
init_temperature: 1
automatic_entropy_tuning: True
model:
_target_: model.rl.gaussian_calql.CalQL_Gaussian
randn_clip_value: 3
cql_min_q_weight: 5.0
tanh_output: True
network_path: ${base_policy_path}
actor:
_target_: model.common.mlp_gaussian.Gaussian_MLP
mlp_dims: [512, 512, 512]
activation_type: ReLU
tanh_output: False # squash after sampling instead
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
horizon_steps: ${horizon_steps}
action_dim: ${action_dim}
std_max: 7.3891
std_min: 0.0067
critic:
_target_: model.common.critic.CriticObsAct
mlp_dims: [256, 256, 256]
activation_type: ReLU
use_layernorm: True
double_q: True
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
action_dim: ${action_dim}
action_steps: ${act_steps}
horizon_steps: ${horizon_steps}
device: ${device}
offline_dataset:
_target_: agent.dataset.sequence.StitchedSequenceQLearningDataset
dataset_path: ${offline_dataset_path}
horizon_steps: ${horizon_steps}
cond_steps: ${cond_steps}
device: ${device}
discount_factor: ${train.gamma}
get_mc_return: True
cfg/robomimic/finetune/square/ft_awr_diffusion_mlp.yaml
+1 -1
View File
@@ -46,7 +46,7 @@ wandb:
run: ${now:%H-%M-%S}_${name}
train:
n_train_itr: 300
n_train_itr: 201
n_critic_warmup_itr: 2
n_steps: 400
gamma: 0.999
cfg/robomimic/finetune/square/ft_dql_diffusion_mlp.yaml
+1 -1
View File
@@ -46,7 +46,7 @@ wandb:
run: ${now:%H-%M-%S}_${name}
train:
n_train_itr: 300
n_train_itr: 201
n_critic_warmup_itr: 5
n_steps: 400
gamma: 0.999
cfg/robomimic/finetune/square/ft_idql_diffusion_mlp.yaml
+1 -1
View File
@@ -46,7 +46,7 @@ wandb:
run: ${now:%H-%M-%S}_${name}
train:
n_train_itr: 300
n_train_itr: 201
n_critic_warmup_itr: 5
n_steps: 400
gamma: 0.999
cfg/robomimic/finetune/square/ft_ppo_diffusion_mlp.yaml
+4 -4
View File
@@ -47,16 +47,16 @@ wandb:
run: ${now:%H-%M-%S}_${name}
train:
n_train_itr: 500
n_train_itr: 201
n_critic_warmup_itr: 2
n_steps: 400
gamma: 0.999
actor_lr: 1e-5
actor_lr: 1e-4
actor_weight_decay: 0
actor_lr_scheduler:
first_cycle_steps: 1000
warmup_steps: 10
min_lr: 1e-5
min_lr: 1e-4
critic_lr: 1e-3
critic_weight_decay: 0
critic_lr_scheduler:
@@ -100,10 +100,10 @@ model:
action_dim: ${action_dim}
critic:
_target_: model.common.critic.CriticObs
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
mlp_dims: [256, 256, 256]
activation_type: Mish
residual_style: True
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
ft_denoising_steps: ${ft_denoising_steps}
horizon_steps: ${horizon_steps}
obs_dim: ${obs_dim}
cfg/robomimic/finetune/square/ft_ppo_diffusion_mlp_img.yaml
+6 -6
View File
@@ -1,7 +1,7 @@
defaults:
- _self_
hydra:
run:
run:
dir: ${logdir}
_target_: agent.finetune.train_ppo_diffusion_img_agent.TrainPPOImgDiffusionAgent
@@ -60,7 +60,7 @@ wandb:
run: ${now:%H-%M-%S}_${name}
train:
n_train_itr: 500
n_train_itr: 301
n_critic_warmup_itr: 2
n_steps: 400
gamma: 0.999
@@ -69,13 +69,13 @@ train:
actor_lr: 1e-5
actor_weight_decay: 0
actor_lr_scheduler:
first_cycle_steps: 500
first_cycle_steps: 1000
warmup_steps: 10
min_lr: 1e-5
critic_lr: 1e-3
critic_weight_decay: 0
critic_lr_scheduler:
first_cycle_steps: 500
first_cycle_steps: 1000
warmup_steps: 10
min_lr: 1e-3
save_model_freq: 100
@@ -96,7 +96,7 @@ train:
model:
_target_: model.diffusion.diffusion_ppo.PPODiffusion
# HP to tune
gamma_denoising: 0.9
gamma_denoising: 0.99
clip_ploss_coef: 0.01
clip_ploss_coef_base: 0.001
clip_ploss_coef_rate: 3
@@ -158,10 +158,10 @@ model:
embed_style: embed2
embed_norm: 0
img_cond_steps: ${img_cond_steps}
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
mlp_dims: [256, 256, 256]
activation_type: Mish
residual_style: True
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
ft_denoising_steps: ${ft_denoising_steps}
horizon_steps: ${horizon_steps}
obs_dim: ${obs_dim}
cfg/robomimic/finetune/square/ft_ppo_diffusion_mlp_ta1.yaml
+112
View File
@@ -0,0 +1,112 @@
defaults:
- _self_
hydra:
run:
dir: ${logdir}
_target_: agent.finetune.train_ppo_diffusion_agent.TrainPPODiffusionAgent
name: ${env_name}_ft_diffusion_mlp_ta${horizon_steps}_td${denoising_steps}_tdf${ft_denoising_steps}
logdir: ${oc.env:DPPO_LOG_DIR}/robomimic-finetune/${name}/${now:%Y-%m-%d}_${now:%H-%M-%S}_${seed}
base_policy_path: ${oc.env:DPPO_LOG_DIR}/robomimic-pretrain/square_pre_diffusion_mlp_ta1_td20/2024-09-29_02-14-14_42/checkpoint/state_8000.pt
robomimic_env_cfg_path: cfg/robomimic/env_meta/${env_name}.json
normalization_path: ${oc.env:DPPO_DATA_DIR}/robomimic/${env_name}/normalization.npz
seed: 42
device: cuda:0
env_name: square
obs_dim: 23
action_dim: 7
denoising_steps: 20
ft_denoising_steps: 10
cond_steps: 1
horizon_steps: 1
act_steps: 1
env:
n_envs: 50
name: ${env_name}
best_reward_threshold_for_success: 1
max_episode_steps: 400
save_video: false
wrappers:
robomimic_lowdim:
normalization_path: ${normalization_path}
low_dim_keys: ['robot0_eef_pos',
'robot0_eef_quat',
'robot0_gripper_qpos',
'object'] # same order of preprocessed observations
multi_step:
n_obs_steps: ${cond_steps}
n_action_steps: ${act_steps}
max_episode_steps: ${env.max_episode_steps}
reset_within_step: True
wandb:
entity: ${oc.env:DPPO_WANDB_ENTITY}
project: robomimic-${env_name}-finetune
run: ${now:%H-%M-%S}_${name}
train:
n_train_itr: 301
n_critic_warmup_itr: 2
n_steps: 400
gamma: 0.999
actor_lr: 1e-4
actor_weight_decay: 0
actor_lr_scheduler:
first_cycle_steps: 1000
warmup_steps: 10
min_lr: 1e-4
critic_lr: 1e-3
critic_weight_decay: 0
critic_lr_scheduler:
first_cycle_steps: 1000
warmup_steps: 10
min_lr: 1e-3
save_model_freq: 100
val_freq: 10
render:
freq: 1
num: 0
# PPO specific
reward_scale_running: True
reward_scale_const: 1.0
gae_lambda: 0.95
batch_size: 20000
update_epochs: 10
vf_coef: 0.5
target_kl: 1
model:
_target_: model.diffusion.diffusion_ppo.PPODiffusion
# HP to tune
gamma_denoising: 0.99
clip_ploss_coef: 0.01
clip_ploss_coef_base: 0.001
clip_ploss_coef_rate: 3
randn_clip_value: 3
min_sampling_denoising_std: 0.1
min_logprob_denoising_std: 0.1
#
network_path: ${base_policy_path}
actor:
_target_: model.diffusion.mlp_diffusion.DiffusionMLP
time_dim: 32
mlp_dims: [1024, 1024, 1024]
cond_mlp_dims: [512, 64]
residual_style: True
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
horizon_steps: ${horizon_steps}
action_dim: ${action_dim}
critic:
_target_: model.common.critic.CriticObs
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
mlp_dims: [256, 256, 256]
activation_type: Mish
residual_style: True
ft_denoising_steps: ${ft_denoising_steps}
horizon_steps: ${horizon_steps}
obs_dim: ${obs_dim}
action_dim: ${action_dim}
denoising_steps: ${denoising_steps}
device: ${device}
cfg/robomimic/finetune/square/ft_ppo_diffusion_mlp_ta1_ph.yaml
+112
View File
@@ -0,0 +1,112 @@
defaults:
- _self_
hydra:
run:
dir: ${logdir}
_target_: agent.finetune.train_ppo_diffusion_agent.TrainPPODiffusionAgent
name: ${env_name}_ft_diffusion_mlp_ta${horizon_steps}_td${denoising_steps}_tdf${ft_denoising_steps}
logdir: ${oc.env:DPPO_LOG_DIR}/robomimic-finetune/${name}/${now:%Y-%m-%d}_${now:%H-%M-%S}_${seed}
base_policy_path: ${oc.env:DPPO_LOG_DIR}/robomimic-pretrain/square_pre_diffusion_mlp_ta1_td20/2024-10-14_10-54-33_0/checkpoint/state_5000.pt
robomimic_env_cfg_path: cfg/robomimic/env_meta/${env_name}.json
normalization_path: ${oc.env:DPPO_DATA_DIR}/robomimic/${env_name}-ph/normalization.npz
seed: 42
device: cuda:0
env_name: square
obs_dim: 23
action_dim: 7
denoising_steps: 20
ft_denoising_steps: 10
cond_steps: 1
horizon_steps: 1
act_steps: 1
env:
n_envs: 40
name: ${env_name}
best_reward_threshold_for_success: 1
max_episode_steps: 400
save_video: false
wrappers:
robomimic_lowdim:
normalization_path: ${normalization_path}
low_dim_keys: ['robot0_eef_pos',
'robot0_eef_quat',
'robot0_gripper_qpos',
'object'] # same order of preprocessed observations
multi_step:
n_obs_steps: ${cond_steps}
n_action_steps: ${act_steps}
max_episode_steps: ${env.max_episode_steps}
reset_within_step: True
wandb:
entity: ${oc.env:DPPO_WANDB_ENTITY}
project: robomimic-${env_name}-finetune
run: ${now:%H-%M-%S}_${name}
train:
n_train_itr: 301
n_critic_warmup_itr: 2
n_steps: 400
gamma: 0.999
actor_lr: 1e-4
actor_weight_decay: 0
actor_lr_scheduler:
first_cycle_steps: 1000
warmup_steps: 10
min_lr: 1e-4
critic_lr: 1e-3
critic_weight_decay: 0
critic_lr_scheduler:
first_cycle_steps: 1000
warmup_steps: 10
min_lr: 1e-3
save_model_freq: 100
val_freq: 10
render:
freq: 1
num: 0
# PPO specific
reward_scale_running: True
reward_scale_const: 1.0
gae_lambda: 0.95
batch_size: 8000
update_epochs: 10
vf_coef: 0.5
target_kl: 1
model:
_target_: model.diffusion.diffusion_ppo.PPODiffusion
# HP to tune
gamma_denoising: 0.9
clip_ploss_coef: 0.01
clip_ploss_coef_base: 0.001
clip_ploss_coef_rate: 3
randn_clip_value: 3
min_sampling_denoising_std: 0.1
min_logprob_denoising_std: 0.1
#
network_path: ${base_policy_path}
actor:
_target_: model.diffusion.mlp_diffusion.DiffusionMLP
time_dim: 32
mlp_dims: [1024, 1024, 1024]
cond_mlp_dims: [512, 64]
residual_style: True
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
horizon_steps: ${horizon_steps}
action_dim: ${action_dim}
critic:
_target_: model.common.critic.CriticObs
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
mlp_dims: [256, 256, 256]
activation_type: Mish
residual_style: True
ft_denoising_steps: ${ft_denoising_steps}
horizon_steps: ${horizon_steps}
obs_dim: ${obs_dim}
action_dim: ${action_dim}
denoising_steps: ${denoising_steps}
device: ${device}
cfg/robomimic/finetune/square/ft_qsm_diffusion_mlp.yaml
+1 -1
View File
@@ -46,7 +46,7 @@ wandb:
run: ${now:%H-%M-%S}_${name}
train:
n_train_itr: 300
n_train_itr: 201
n_critic_warmup_itr: 5
n_steps: 400
gamma: 0.999
cfg/robomimic/finetune/square/ft_rwr_diffusion_mlp.yaml
+1 -1
View File
@@ -46,7 +46,7 @@ wandb:
run: ${now:%H-%M-%S}_${name}
train:
n_train_itr: 300
n_train_itr: 201
n_critic_warmup_itr: 2
n_steps: 400
gamma: 0.999
cfg/robomimic/finetune/square/ibrl_mlp.yaml
+2 -2
View File
@@ -7,7 +7,7 @@ _target_: agent.finetune.train_ibrl_agent.TrainIBRLAgent
name: ${env_name}_ibrl_mlp_ta${horizon_steps}
logdir: ${oc.env:DPPO_LOG_DIR}/robomimic-finetune/${name}/${now:%Y-%m-%d}_${now:%H-%M-%S}_${seed}
base_policy_path:
base_policy_path: ${oc.env:DPPO_LOG_DIR}/robomimic-pretrain/square_pre_gaussian_mlp_ta1/2024-09-28_13-42-43_42/checkpoint/state_5000.pt
robomimic_env_cfg_path: cfg/robomimic/env_meta/${env_name}.json
normalization_path: ${oc.env:DPPO_DATA_DIR}/robomimic/${env_name}/normalization.npz
offline_dataset_path: ${oc.env:DPPO_DATA_DIR}/robomimic/${env_name}/train.npz
@@ -93,7 +93,7 @@ model:
fixed_std: 0.1
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
horizon_steps: ${horizon_steps}
action_dim: ${action_dim}
critic:
_target_: model.common.critic.CriticObsAct
mlp_dims: [1024, 1024, 1024]
cfg/robomimic/finetune/square/ibrl_mlp_ph.yaml
+115
View File
@@ -0,0 +1,115 @@
defaults:
- _self_
hydra:
run:
dir: ${logdir}
_target_: agent.finetune.train_ibrl_agent.TrainIBRLAgent
name: ${env_name}_ibrl_mlp_ta${horizon_steps}
logdir: ${oc.env:DPPO_LOG_DIR}/robomimic-finetune/${name}/${now:%Y-%m-%d}_${now:%H-%M-%S}_${seed}
base_policy_path: ${oc.env:DPPO_LOG_DIR}/robomimic-pretrain/square_pre_gaussian_mlp_ta1/2024-10-08_20-52-42_0/checkpoint/state_5000.pt
robomimic_env_cfg_path: cfg/robomimic/env_meta/${env_name}.json
normalization_path: ${oc.env:DPPO_DATA_DIR}/robomimic/${env_name}-ph/normalization.npz
offline_dataset_path: ${oc.env:DPPO_DATA_DIR}/robomimic/${env_name}-ph/train.npz
seed: 42
device: cuda:0
env_name: square
obs_dim: 23
action_dim: 7
cond_steps: 1
horizon_steps: 1
act_steps: 1
env:
n_envs: 1
name: ${env_name}
max_episode_steps: 350 # IBRL uses 300
reset_at_iteration: False
save_video: False
best_reward_threshold_for_success: 1
wrappers:
robomimic_lowdim:
normalization_path: ${normalization_path}
low_dim_keys: ['robot0_eef_pos',
'robot0_eef_quat',
'robot0_gripper_qpos',
'object'] # same order of preprocessed observations
multi_step:
n_obs_steps: ${cond_steps}
n_action_steps: ${act_steps}
max_episode_steps: ${env.max_episode_steps}
reset_within_step: True
wandb:
entity: ${oc.env:DPPO_WANDB_ENTITY}
project: ibrl-${env_name}
run: ${now:%H-%M-%S}_${name}
train:
n_train_itr: 1000000
n_steps: 1
gamma: 0.99
actor_lr: 1e-4
actor_weight_decay: 0
actor_lr_scheduler:
first_cycle_steps: 1000
warmup_steps: 10
min_lr: 1e-4
critic_lr: 1e-4
critic_weight_decay: 0
critic_lr_scheduler:
first_cycle_steps: 1000
warmup_steps: 10
min_lr: 1e-4
save_model_freq: 100000
val_freq: 10000
render:
freq: 10000
num: 0
log_freq: 200
# IBRL specific
batch_size: 256
target_ema_rate: 0.01
scale_reward_factor: 1
critic_num_update: 3
buffer_size: 400000
n_eval_episode: 40
n_explore_steps: 0
update_freq: 2
model:
_target_: model.rl.gaussian_ibrl.IBRL_Gaussian
randn_clip_value: 3
n_critics: 5
soft_action_sample: True
soft_action_sample_beta: 10
network_path: ${base_policy_path}
actor:
_target_: model.common.mlp_gaussian.Gaussian_MLP
mlp_dims: [1024, 1024, 1024]
activation_type: ReLU
dropout: 0.5
fixed_std: 0.1
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
horizon_steps: ${horizon_steps}
action_dim: ${action_dim}
critic:
_target_: model.common.critic.CriticObsAct
mlp_dims: [1024, 1024, 1024]
activation_type: ReLU
use_layernorm: True
double_q: False # use ensemble
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
action_dim: ${action_dim}
action_steps: ${act_steps}
horizon_steps: ${horizon_steps}
device: ${device}
offline_dataset:
_target_: agent.dataset.sequence.StitchedSequenceQLearningDataset
dataset_path: ${offline_dataset_path}
horizon_steps: ${horizon_steps}
cond_steps: ${cond_steps}
device: ${device}
max_n_episodes: 100
cfg/robomimic/finetune/transport/ft_awr_diffusion_mlp.yaml
+4 -4
View File
@@ -26,7 +26,7 @@ env:
name: ${env_name}
best_reward_threshold_for_success: 1
max_episode_steps: 800
save_video: false
save_video: False
wrappers:
robomimic_lowdim:
normalization_path: ${normalization_path}
@@ -49,7 +49,7 @@ wandb:
run: ${now:%H-%M-%S}_${name}
train:
n_train_itr: 1000
n_train_itr: 201
n_critic_warmup_itr: 2
n_steps: 400
gamma: 0.999
@@ -58,7 +58,7 @@ train:
actor_lr_scheduler:
first_cycle_steps: 1000
warmup_steps: 10
min_lr: 1e-6
min_lr: 1e-5
critic_lr: 1e-3
critic_weight_decay: 0
critic_lr_scheduler:
@@ -82,7 +82,7 @@ train:
model:
_target_: model.diffusion.diffusion_awr.AWRDiffusion
# Sampling HPs
min_sampling_denoising_std: 0.08
min_sampling_denoising_std: 0.1
randn_clip_value: 3
#
network_path: ${base_policy_path}
cfg/robomimic/finetune/transport/ft_dipo_diffusion_mlp.yaml
+6 -6
View File
@@ -49,7 +49,7 @@ wandb:
run: ${now:%H-%M-%S}_${name}
train:
n_train_itr: 1000
n_train_itr: 201
n_critic_warmup_itr: 2
n_steps: 400
gamma: 0.999
@@ -58,7 +58,7 @@ train:
actor_lr_scheduler:
first_cycle_steps: 1000
warmup_steps: 10
min_lr: 1e-6
min_lr: 1e-5
critic_lr: 1e-3
critic_weight_decay: 0
critic_lr_scheduler:
@@ -82,7 +82,7 @@ train:
model:
_target_: model.diffusion.diffusion_dipo.DIPODiffusion
# HP to tune
min_sampling_denoising_std: 0.08
min_sampling_denoising_std: 0.1
randn_clip_value: 3
#
network_path: ${base_policy_path}
@@ -96,12 +96,12 @@ model:
action_dim: ${action_dim}
critic:
_target_: model.common.critic.CriticObsAct
action_dim: ${action_dim}
action_steps: ${act_steps}
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
mlp_dims: [256, 256, 256]
activation_type: Mish
residual_style: True
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
action_dim: ${action_dim}
action_steps: ${act_steps}
horizon_steps: ${horizon_steps}
obs_dim: ${obs_dim}
action_dim: ${action_dim}
cfg/robomimic/finetune/transport/ft_dql_diffusion_mlp.yaml
+5 -5
View File
@@ -26,7 +26,7 @@ env:
name: ${env_name}
best_reward_threshold_for_success: 1
max_episode_steps: 800
save_video: false
save_video: False
wrappers:
robomimic_lowdim:
normalization_path: ${normalization_path}
@@ -49,8 +49,8 @@ wandb:
run: ${now:%H-%M-%S}_${name}
train:
n_train_itr: 1000
n_critic_warmup_itr: 2
n_train_itr: 201
n_critic_warmup_itr: 5
n_steps: 400
gamma: 0.999
actor_lr: 1e-5
@@ -58,7 +58,7 @@ train:
actor_lr_scheduler:
first_cycle_steps: 1000
warmup_steps: 10
min_lr: 1e-6
min_lr: 1e-5
critic_lr: 1e-3
critic_weight_decay: 0
critic_lr_scheduler:
@@ -81,7 +81,7 @@ train:
model:
_target_: model.diffusion.diffusion_dql.DQLDiffusion
# Sampling HPs
min_sampling_denoising_std: 0.08
min_sampling_denoising_std: 0.1
randn_clip_value: 3
#
network_path: ${base_policy_path}
cfg/robomimic/finetune/transport/ft_idql_diffusion_mlp.yaml
+3 -3
View File
@@ -49,7 +49,7 @@ wandb:
run: ${now:%H-%M-%S}_${name}
train:
n_train_itr: 1000
n_train_itr: 201
n_critic_warmup_itr: 5
n_steps: 400
gamma: 0.999
@@ -58,7 +58,7 @@ train:
actor_lr_scheduler:
first_cycle_steps: 1000
warmup_steps: 10
min_lr: 1e-6
min_lr: 1e-5
critic_lr: 1e-3
critic_weight_decay: 0
critic_lr_scheduler:
@@ -83,7 +83,7 @@ train:
model:
_target_: model.diffusion.diffusion_idql.IDQLDiffusion
# Sampling HPs
min_sampling_denoising_std: 0.08
min_sampling_denoising_std: 0.1
randn_clip_value: 3
#
network_path: ${base_policy_path}
cfg/robomimic/finetune/transport/ft_ppo_diffusion_mlp.yaml
+6 -6
View File
@@ -50,16 +50,16 @@ wandb:
run: ${now:%H-%M-%S}_${name}
train:
n_train_itr: 1000
n_train_itr: 201
n_critic_warmup_itr: 2
n_steps: 400
gamma: 0.999
actor_lr: 1e-5
actor_lr: 1e-4
actor_weight_decay: 0
actor_lr_scheduler:
first_cycle_steps: 1000
warmup_steps: 10
min_lr: 1e-6
min_lr: 1e-4
critic_lr: 1e-3
critic_weight_decay: 0
critic_lr_scheduler:
@@ -76,7 +76,7 @@ train:
reward_scale_const: 1.0
gae_lambda: 0.95
batch_size: 10000
update_epochs: 8
update_epochs: 5
vf_coef: 0.5
target_kl: 1
@@ -88,7 +88,7 @@ model:
clip_ploss_coef_base: 0.001
clip_ploss_coef_rate: 3
randn_clip_value: 3
min_sampling_denoising_std: 0.08
min_sampling_denoising_std: 0.1
min_logprob_denoising_std: 0.1
#
network_path: ${base_policy_path}
@@ -102,10 +102,10 @@ model:
action_dim: ${action_dim}
critic:
_target_: model.common.critic.CriticObs
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
mlp_dims: [256, 256, 256]
activation_type: Mish
residual_style: True
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
ft_denoising_steps: ${ft_denoising_steps}
horizon_steps: ${horizon_steps}
obs_dim: ${obs_dim}
cfg/robomimic/finetune/transport/ft_ppo_diffusion_mlp_img.yaml
+9 -9
View File
@@ -1,7 +1,7 @@
defaults:
- _self_
hydra:
run:
run:
dir: ${logdir}
_target_: agent.finetune.train_ppo_diffusion_img_agent.TrainPPOImgDiffusionAgent
@@ -64,7 +64,7 @@ wandb:
run: ${now:%H-%M-%S}_${name}
train:
n_train_itr: 500
n_train_itr: 201
n_critic_warmup_itr: 2
n_steps: 400
gamma: 0.999
@@ -73,13 +73,13 @@ train:
actor_lr: 1e-5
actor_weight_decay: 0
actor_lr_scheduler:
first_cycle_steps: 500
first_cycle_steps: 1000
warmup_steps: 10
min_lr: 1e-6
min_lr: 1e-5
critic_lr: 1e-3
critic_weight_decay: 0
critic_lr_scheduler:
first_cycle_steps: 500
first_cycle_steps: 1000
warmup_steps: 10
min_lr: 1e-3
save_model_freq: 100
@@ -93,19 +93,19 @@ train:
gae_lambda: 0.95
batch_size: 500
logprob_batch_size: 1000
update_epochs: 8
update_epochs: 10
vf_coef: 0.5
target_kl: 1
model:
_target_: model.diffusion.diffusion_ppo.PPODiffusion
# HP to tune
gamma_denoising: 0.9
gamma_denoising: 0.99
clip_ploss_coef: 0.01
clip_ploss_coef_base: 0.001
clip_ploss_coef_rate: 3
randn_clip_value: 3
min_sampling_denoising_std: 0.08
min_sampling_denoising_std: 0.1
min_logprob_denoising_std: 0.1
#
use_ddim: ${use_ddim}
@@ -164,10 +164,10 @@ model:
embed_style: embed2
embed_norm: 0
img_cond_steps: ${img_cond_steps}
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
mlp_dims: [256, 256, 256]
activation_type: Mish
residual_style: True
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
ft_denoising_steps: ${ft_denoising_steps}
horizon_steps: ${horizon_steps}
obs_dim: ${obs_dim}
cfg/robomimic/finetune/transport/ft_qsm_diffusion_mlp.yaml
+3 -3
View File
@@ -49,7 +49,7 @@ wandb:
run: ${now:%H-%M-%S}_${name}
train:
n_train_itr: 1000
n_train_itr: 201
n_critic_warmup_itr: 5
n_steps: 400
gamma: 0.999
@@ -58,7 +58,7 @@ train:
actor_lr_scheduler:
first_cycle_steps: 1000
warmup_steps: 10
min_lr: 1e-6
min_lr: 1e-5
critic_lr: 1e-3
critic_weight_decay: 0
critic_lr_scheduler:
@@ -81,7 +81,7 @@ train:
model:
_target_: model.diffusion.diffusion_qsm.QSMDiffusion
# Sampling HPs
min_sampling_denoising_std: 0.08
min_sampling_denoising_std: 0.1
randn_clip_value: 3
#
network_path: ${base_policy_path}
cfg/robomimic/finetune/transport/ft_rwr_diffusion_mlp.yaml
+3 -3
View File
@@ -49,7 +49,7 @@ wandb:
run: ${now:%H-%M-%S}_${name}
train:
n_train_itr: 1000
n_train_itr: 201
n_critic_warmup_itr: 2
n_steps: 400
gamma: 0.999
@@ -58,7 +58,7 @@ train:
lr_scheduler:
first_cycle_steps: 1000
warmup_steps: 10
min_lr: 1e-6
min_lr: 1e-5
save_model_freq: 100
val_freq: 10
render:
@@ -73,7 +73,7 @@ train:
model:
_target_: model.diffusion.diffusion_rwr.RWRDiffusion
# Sampling HPs
min_sampling_denoising_std: 0.08
min_sampling_denoising_std: 0.1
randn_clip_value: 3
#
network_path: ${base_policy_path}
cfg/robomimic/pretrain/can/calql_mlp_offline.yaml
+5 -5
View File
@@ -46,7 +46,7 @@ wandb:
run: ${now:%H-%M-%S}_${name}
train:
n_train_itr: 100
n_train_itr: 1000
n_steps: 1
gamma: 0.99
actor_lr: 1e-4
@@ -61,8 +61,8 @@ train:
first_cycle_steps: 1000
warmup_steps: 10
min_lr: 3e-4
save_model_freq: 10
val_freq: 10
save_model_freq: 100
val_freq: 20
render:
freq: 1
num: 0
@@ -70,7 +70,7 @@ train:
# CalQL specific
train_online: False
batch_size: 256
n_random_actions: 4
n_random_actions: 10
target_ema_rate: 0.005
scale_reward_factor: 1.0
num_update: 1000
@@ -93,7 +93,7 @@ model:
tanh_output: False # squash after sampling instead
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
horizon_steps: ${horizon_steps}
action_dim: ${action_dim}
std_max: 7.3891
std_min: 0.0067
critic:
cfg/robomimic/pretrain/can/calql_mlp_offline_ph.yaml
+118
View File
@@ -0,0 +1,118 @@
defaults:
- _self_
hydra:
run:
dir: ${logdir}
_target_: agent.finetune.train_calql_agent.TrainCalQLAgent
name: ${env_name}_calql_mlp_ta${horizon_steps}
logdir: ${oc.env:DPPO_LOG_DIR}/robomimic-pretrain/${name}/${now:%Y-%m-%d}_${now:%H-%M-%S}_${seed}
robomimic_env_cfg_path: cfg/robomimic/env_meta/${env_name}.json
normalization_path: ${oc.env:DPPO_DATA_DIR}/robomimic/${env_name}-ph/normalization.npz
offline_dataset_path: ${oc.env:DPPO_DATA_DIR}/robomimic/${env_name}-ph/train.npz
seed: 42
device: cuda:0
env_name: can
obs_dim: 23
action_dim: 7
cond_steps: 1
horizon_steps: 1
act_steps: 1
env:
n_envs: 1
name: ${env_name}
best_reward_threshold_for_success: 1
max_episode_steps: 300
reset_at_iteration: False
save_video: False
wrappers:
robomimic_lowdim:
normalization_path: ${normalization_path}
low_dim_keys: ['robot0_eef_pos',
'robot0_eef_quat',
'robot0_gripper_qpos',
'object'] # same order of preprocessed observations
multi_step:
n_obs_steps: ${cond_steps}
n_action_steps: ${act_steps}
max_episode_steps: ${env.max_episode_steps}
reset_within_step: True
wandb:
entity: ${oc.env:DPPO_WANDB_ENTITY}
project: calql-${env_name}
run: ${now:%H-%M-%S}_${name}
train:
n_train_itr: 1000
n_steps: 1
gamma: 0.99
actor_lr: 1e-4
actor_weight_decay: 0
actor_lr_scheduler:
first_cycle_steps: 1000
warmup_steps: 10
min_lr: 1e-4
critic_lr: 3e-4
critic_weight_decay: 0
critic_lr_scheduler:
first_cycle_steps: 1000
warmup_steps: 10
min_lr: 3e-4
save_model_freq: 10
val_freq: 10
render:
freq: 1
num: 0
log_freq: 1
# CalQL specific
train_online: False
batch_size: 256
n_random_actions: 4
target_ema_rate: 0.005
scale_reward_factor: 1.0
num_update: 1000
buffer_size: 1000000
n_eval_episode: 10
n_explore_steps: 0
target_entropy: ${eval:'- ${action_dim} * ${act_steps}'}
init_temperature: 1
automatic_entropy_tuning: True
model:
_target_: model.rl.gaussian_calql.CalQL_Gaussian
randn_clip_value: 3
cql_min_q_weight: 5.0
tanh_output: True
actor:
_target_: model.common.mlp_gaussian.Gaussian_MLP
mlp_dims: [512, 512, 512]
activation_type: ReLU
tanh_output: False # squash after sampling instead
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
horizon_steps: ${horizon_steps}
action_dim: ${action_dim}
std_max: 7.3891
std_min: 0.0067
critic:
_target_: model.common.critic.CriticObsAct
mlp_dims: [256, 256, 256]
activation_type: ReLU
use_layernorm: True
double_q: True
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
action_dim: ${action_dim}
action_steps: ${act_steps}
horizon_steps: ${horizon_steps}
device: ${device}
offline_dataset:
_target_: agent.dataset.sequence.StitchedSequenceQLearningDataset
dataset_path: ${offline_dataset_path}
horizon_steps: ${horizon_steps}
cond_steps: ${cond_steps}
device: ${device}
discount_factor: ${train.gamma}
get_mc_return: True
cfg/robomimic/pretrain/can/pre_diffusion_mlp_ta1.yaml
+65
View File
@@ -0,0 +1,65 @@
defaults:
- _self_
hydra:
run:
dir: ${logdir}
_target_: agent.pretrain.train_diffusion_agent.TrainDiffusionAgent
name: ${env}_pre_diffusion_mlp_ta${horizon_steps}_td${denoising_steps}
logdir: ${oc.env:DPPO_LOG_DIR}/robomimic-pretrain/${name}/${now:%Y-%m-%d}_${now:%H-%M-%S}_${seed}
train_dataset_path: ${oc.env:DPPO_DATA_DIR}/robomimic/${env}/train.npz
seed: 42
device: cuda:0
env: can
obs_dim: 23
action_dim: 7
denoising_steps: 20
horizon_steps: 1
cond_steps: 1
wandb:
entity: ${oc.env:DPPO_WANDB_ENTITY}
project: robomimic-${env}-pretrain
run: ${now:%H-%M-%S}_${name}
train:
n_epochs: 8000
batch_size: 256
learning_rate: 1e-4
weight_decay: 1e-6
lr_scheduler:
first_cycle_steps: 10000
warmup_steps: 100
min_lr: 1e-5
epoch_start_ema: 20
update_ema_freq: 10
save_model_freq: 1000
model:
_target_: model.diffusion.diffusion.DiffusionModel
predict_epsilon: True
denoised_clip_value: 1.0
network:
_target_: model.diffusion.mlp_diffusion.DiffusionMLP
time_dim: 16
mlp_dims: [512, 512, 512]
residual_style: True
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
horizon_steps: ${horizon_steps}
action_dim: ${action_dim}
horizon_steps: ${horizon_steps}
obs_dim: ${obs_dim}
action_dim: ${action_dim}
denoising_steps: ${denoising_steps}
device: ${device}
ema:
decay: 0.995
train_dataset:
_target_: agent.dataset.sequence.StitchedSequenceDataset
dataset_path: ${train_dataset_path}
horizon_steps: ${horizon_steps}
cond_steps: ${cond_steps}
device: ${device}
cfg/robomimic/pretrain/can/pre_diffusion_mlp_ta1_ph.yaml
+65
View File
@@ -0,0 +1,65 @@
defaults:
- _self_
hydra:
run:
dir: ${logdir}
_target_: agent.pretrain.train_diffusion_agent.TrainDiffusionAgent
name: ${env}_pre_diffusion_mlp_ta${horizon_steps}_td${denoising_steps}
logdir: ${oc.env:DPPO_LOG_DIR}/robomimic-pretrain/${name}/${now:%Y-%m-%d}_${now:%H-%M-%S}_${seed}
train_dataset_path: ${oc.env:DPPO_DATA_DIR}/robomimic/${env}-ph/train.npz
seed: 42
device: cuda:0
env: can
obs_dim: 23
action_dim: 7
denoising_steps: 20
horizon_steps: 1
cond_steps: 1
wandb:
entity: ${oc.env:DPPO_WANDB_ENTITY}
project: robomimic-${env}-pretrain
run: ${now:%H-%M-%S}_${name}
train:
n_epochs: 8000
batch_size: 256
learning_rate: 1e-4
weight_decay: 1e-6
lr_scheduler:
first_cycle_steps: 10000
warmup_steps: 100
min_lr: 1e-5
epoch_start_ema: 20
update_ema_freq: 10
save_model_freq: 1000
model:
_target_: model.diffusion.diffusion.DiffusionModel
predict_epsilon: True
denoised_clip_value: 1.0
network:
_target_: model.diffusion.mlp_diffusion.DiffusionMLP
time_dim: 16
mlp_dims: [512, 512, 512]
residual_style: True
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
horizon_steps: ${horizon_steps}
action_dim: ${action_dim}
horizon_steps: ${horizon_steps}
obs_dim: ${obs_dim}
action_dim: ${action_dim}
denoising_steps: ${denoising_steps}
device: ${device}
ema:
decay: 0.995
train_dataset:
_target_: agent.dataset.sequence.StitchedSequenceDataset
dataset_path: ${train_dataset_path}
horizon_steps: ${horizon_steps}
cond_steps: ${cond_steps}
device: ${device}
cfg/robomimic/pretrain/transport/pre_gaussian_mlp_ibrl.yaml → cfg/robomimic/pretrain/can/pre_gaussian_mlp_ta1_ph.yaml
+7 -7
View File
@@ -7,13 +7,13 @@ _target_: agent.pretrain.train_gaussian_agent.TrainGaussianAgent
name: ${env}_pre_gaussian_mlp_ta${horizon_steps}
logdir: ${oc.env:DPPO_LOG_DIR}/robomimic-pretrain/${name}/${now:%Y-%m-%d}_${now:%H-%M-%S}_${seed}
train_dataset_path: ${oc.env:DPPO_DATA_DIR}/robomimic/${env}/train.npz
train_dataset_path: ${oc.env:DPPO_DATA_DIR}/robomimic/${env}-ph/train.npz
seed: 42
device: cuda:0
env: transport
obs_dim: 59
action_dim: 14
env: can
obs_dim: 23
action_dim: 7
horizon_steps: 1
cond_steps: 1
@@ -26,11 +26,11 @@ train:
n_epochs: 5000
batch_size: 256
learning_rate: 1e-4
weight_decay: 0
weight_decay: 1e-6
lr_scheduler:
first_cycle_steps: 5000
warmup_steps: 100
min_lr: 1e-4
min_lr: 1e-5
epoch_start_ema: 20
update_ema_freq: 10
save_model_freq: 1000
@@ -45,7 +45,7 @@ model:
fixed_std: 0.1
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
horizon_steps: ${horizon_steps}
action_dim: ${action_dim}
action_dim: ${action_dim}
horizon_steps: ${horizon_steps}
device: ${device}
cfg/robomimic/pretrain/square/calql_mlp_offline.yaml
+5 -5
View File
@@ -46,7 +46,7 @@ wandb:
run: ${now:%H-%M-%S}_${name}
train:
n_train_itr: 100
n_train_itr: 1000
n_steps: 1
gamma: 0.99
actor_lr: 1e-4
@@ -61,8 +61,8 @@ train:
first_cycle_steps: 1000
warmup_steps: 10
min_lr: 3e-4
save_model_freq: 10
val_freq: 10
save_model_freq: 100
val_freq: 20
render:
freq: 1
num: 0
@@ -70,7 +70,7 @@ train:
# CalQL specific
train_online: False
batch_size: 256
n_random_actions: 4
n_random_actions: 10
target_ema_rate: 0.005
scale_reward_factor: 1.0
num_update: 1000
@@ -93,7 +93,7 @@ model:
tanh_output: False # squash after sampling instead
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
horizon_steps: ${horizon_steps}
action_dim: ${action_dim}
std_max: 7.3891
std_min: 0.0067
critic:
cfg/robomimic/pretrain/square/calql_mlp_offline_ph.yaml
+118
View File
@@ -0,0 +1,118 @@
defaults:
- _self_
hydra:
run:
dir: ${logdir}
_target_: agent.finetune.train_calql_agent.TrainCalQLAgent
name: ${env_name}_calql_mlp_ta${horizon_steps}
logdir: ${oc.env:DPPO_LOG_DIR}/robomimic-pretrain/${name}/${now:%Y-%m-%d}_${now:%H-%M-%S}_${seed}
robomimic_env_cfg_path: cfg/robomimic/env_meta/${env_name}.json
normalization_path: ${oc.env:DPPO_DATA_DIR}/robomimic/${env_name}-ph/normalization.npz
offline_dataset_path: ${oc.env:DPPO_DATA_DIR}/robomimic/${env_name}-ph/train.npz
seed: 42
device: cuda:0
env_name: square
obs_dim: 23
action_dim: 7
cond_steps: 1
horizon_steps: 1
act_steps: 1
env:
n_envs: 1
name: ${env_name}
best_reward_threshold_for_success: 1
max_episode_steps: 400
reset_at_iteration: False
save_video: False
wrappers:
robomimic_lowdim:
normalization_path: ${normalization_path}
low_dim_keys: ['robot0_eef_pos',
'robot0_eef_quat',
'robot0_gripper_qpos',
'object'] # same order of preprocessed observations
multi_step:
n_obs_steps: ${cond_steps}
n_action_steps: ${act_steps}
max_episode_steps: ${env.max_episode_steps}
reset_within_step: True
wandb:
entity: ${oc.env:DPPO_WANDB_ENTITY}
project: calql-${env_name}
run: ${now:%H-%M-%S}_${name}
train:
n_train_itr: 1000
n_steps: 1
gamma: 0.99
actor_lr: 1e-4
actor_weight_decay: 0
actor_lr_scheduler:
first_cycle_steps: 1000
warmup_steps: 10
min_lr: 1e-4
critic_lr: 3e-4
critic_weight_decay: 0
critic_lr_scheduler:
first_cycle_steps: 1000
warmup_steps: 10
min_lr: 3e-4
save_model_freq: 10
val_freq: 10
render:
freq: 1
num: 0
log_freq: 1
# CalQL specific
train_online: False
batch_size: 256
n_random_actions: 4
target_ema_rate: 0.005
scale_reward_factor: 1.0
num_update: 1000
buffer_size: 1000000
n_eval_episode: 10
n_explore_steps: 0
target_entropy: ${eval:'- ${action_dim} * ${act_steps}'}
init_temperature: 1
automatic_entropy_tuning: True
model:
_target_: model.rl.gaussian_calql.CalQL_Gaussian
randn_clip_value: 3
cql_min_q_weight: 5.0
tanh_output: True
actor:
_target_: model.common.mlp_gaussian.Gaussian_MLP
mlp_dims: [512, 512, 512]
activation_type: ReLU
tanh_output: False # squash after sampling instead
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
horizon_steps: ${horizon_steps}
action_dim: ${action_dim}
std_max: 7.3891
std_min: 0.0067
critic:
_target_: model.common.critic.CriticObsAct
mlp_dims: [256, 256, 256]
activation_type: ReLU
use_layernorm: True
double_q: True
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
action_dim: ${action_dim}
action_steps: ${act_steps}
horizon_steps: ${horizon_steps}
device: ${device}
offline_dataset:
_target_: agent.dataset.sequence.StitchedSequenceQLearningDataset
dataset_path: ${offline_dataset_path}
horizon_steps: ${horizon_steps}
cond_steps: ${cond_steps}
device: ${device}
discount_factor: ${train.gamma}
get_mc_return: True
cfg/robomimic/pretrain/square/pre_diffusion_mlp_ta1.yaml
+66
View File
@@ -0,0 +1,66 @@
defaults:
- _self_
hydra:
run:
dir: ${logdir}
_target_: agent.pretrain.train_diffusion_agent.TrainDiffusionAgent
name: ${env}_pre_diffusion_mlp_ta${horizon_steps}_td${denoising_steps}
logdir: ${oc.env:DPPO_LOG_DIR}/robomimic-pretrain/${name}/${now:%Y-%m-%d}_${now:%H-%M-%S}_${seed}
train_dataset_path: ${oc.env:DPPO_DATA_DIR}/robomimic/${env}/train.npz
seed: 42
device: cuda:0
env: square
obs_dim: 23
action_dim: 7
denoising_steps: 20
horizon_steps: 1
cond_steps: 1
wandb:
entity: ${oc.env:DPPO_WANDB_ENTITY}
project: robomimic-${env}-pretrain
run: ${now:%H-%M-%S}_${name}
train:
n_epochs: 8000
batch_size: 256
learning_rate: 1e-4
weight_decay: 1e-6
lr_scheduler:
first_cycle_steps: 10000
warmup_steps: 100
min_lr: 1e-5
epoch_start_ema: 20
update_ema_freq: 10
save_model_freq: 1000
model:
_target_: model.diffusion.diffusion.DiffusionModel
predict_epsilon: True
denoised_clip_value: 1.0
network:
_target_: model.diffusion.mlp_diffusion.DiffusionMLP
time_dim: 32
mlp_dims: [1024, 1024, 1024]
cond_mlp_dims: [512, 64]
residual_style: True
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
horizon_steps: ${horizon_steps}
action_dim: ${action_dim}
horizon_steps: ${horizon_steps}
obs_dim: ${obs_dim}
action_dim: ${action_dim}
denoising_steps: ${denoising_steps}
device: ${device}
ema:
decay: 0.995
train_dataset:
_target_: agent.dataset.sequence.StitchedSequenceDataset
dataset_path: ${train_dataset_path}
horizon_steps: ${horizon_steps}
cond_steps: ${cond_steps}
device: ${device}
cfg/robomimic/pretrain/square/pre_diffusion_mlp_ta1_ph.yaml
+66
View File
@@ -0,0 +1,66 @@
defaults:
- _self_
hydra:
run:
dir: ${logdir}
_target_: agent.pretrain.train_diffusion_agent.TrainDiffusionAgent
name: ${env}_pre_diffusion_mlp_ta${horizon_steps}_td${denoising_steps}
logdir: ${oc.env:DPPO_LOG_DIR}/robomimic-pretrain/${name}/${now:%Y-%m-%d}_${now:%H-%M-%S}_${seed}
train_dataset_path: ${oc.env:DPPO_DATA_DIR}/robomimic/${env}-ph/train.npz
seed: 42
device: cuda:0
env: square
obs_dim: 23
action_dim: 7
denoising_steps: 20
horizon_steps: 1
cond_steps: 1
wandb:
entity: ${oc.env:DPPO_WANDB_ENTITY}
project: robomimic-${env}-pretrain
run: ${now:%H-%M-%S}_${name}
train:
n_epochs: 8000
batch_size: 256
learning_rate: 1e-4
weight_decay: 1e-6
lr_scheduler:
first_cycle_steps: 10000
warmup_steps: 100
min_lr: 1e-5
epoch_start_ema: 20
update_ema_freq: 10
save_model_freq: 1000
model:
_target_: model.diffusion.diffusion.DiffusionModel
predict_epsilon: True
denoised_clip_value: 1.0
network:
_target_: model.diffusion.mlp_diffusion.DiffusionMLP
time_dim: 32
mlp_dims: [1024, 1024, 1024]
cond_mlp_dims: [512, 64]
residual_style: True
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
horizon_steps: ${horizon_steps}
action_dim: ${action_dim}
horizon_steps: ${horizon_steps}
obs_dim: ${obs_dim}
action_dim: ${action_dim}
denoising_steps: ${denoising_steps}
device: ${device}
ema:
decay: 0.995
train_dataset:
_target_: agent.dataset.sequence.StitchedSequenceDataset
dataset_path: ${train_dataset_path}
horizon_steps: ${horizon_steps}
cond_steps: ${cond_steps}
device: ${device}
cfg/robomimic/pretrain/lift/pre_gaussian_mlp_ibrl.yaml → cfg/robomimic/pretrain/square/pre_gaussian_mlp_ta1_ph.yaml
+7 -6
View File
@@ -7,12 +7,12 @@ _target_: agent.pretrain.train_gaussian_agent.TrainGaussianAgent
name: ${env}_pre_gaussian_mlp_ta${horizon_steps}
logdir: ${oc.env:DPPO_LOG_DIR}/robomimic-pretrain/${name}/${now:%Y-%m-%d}_${now:%H-%M-%S}_${seed}
train_dataset_path: ${oc.env:DPPO_DATA_DIR}/robomimic/${env}/train.npz
train_dataset_path: ${oc.env:DPPO_DATA_DIR}/robomimic/${env}-ph/train.npz
seed: 42
device: cuda:0
env: lift
obs_dim: 19
env: square
obs_dim: 23
action_dim: 7
horizon_steps: 1
cond_steps: 1
@@ -40,14 +40,15 @@ model:
network:
_target_: model.common.mlp_gaussian.Gaussian_MLP
mlp_dims: [1024, 1024, 1024]
residual_style: False
activation_type: ReLU
dropout: 0.5
fixed_std: 0.1
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
horizon_steps: ${horizon_steps}
action_dim: ${action_dim}
action_dim: ${action_dim}
horizon_steps: ${horizon_steps}
device: ${device}
ema:
decay: 0.995
cfg/robomimic/finetune/can/rlpd_mlp.yaml → cfg/robomimic/scratch/can/rlpd_mlp.yaml
+1 -1
View File
@@ -91,7 +91,7 @@ model:
tanh_output: False # squash after sampling instead
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
horizon_steps: ${horizon_steps}
action_dim: ${action_dim}
std_max: 7.3891
std_min: 0.0067
critic:
cfg/robomimic/scratch/can/rlpd_mlp_ph.yaml
+114
View File
@@ -0,0 +1,114 @@
defaults:
- _self_
hydra:
run:
dir: ${logdir}
_target_: agent.finetune.train_rlpd_agent.TrainRLPDAgent
name: ${env_name}_rlpd_mlp_ta${horizon_steps}
logdir: ${oc.env:DPPO_LOG_DIR}/robomimic-finetune/${name}/${now:%Y-%m-%d}_${now:%H-%M-%S}_${seed}
robomimic_env_cfg_path: cfg/robomimic/env_meta/${env_name}.json
normalization_path: ${oc.env:DPPO_DATA_DIR}/robomimic/${env_name}-ph/normalization.npz
offline_dataset_path: ${oc.env:DPPO_DATA_DIR}/robomimic/${env_name}-ph/train.npz
seed: 42
device: cuda:0
env_name: can
obs_dim: 23
action_dim: 7
cond_steps: 1
horizon_steps: 1
act_steps: 1
env:
n_envs: 1
name: ${env_name}
max_episode_steps: 300
reset_at_iteration: False
save_video: False
best_reward_threshold_for_success: 1
wrappers:
robomimic_lowdim:
normalization_path: ${normalization_path}
low_dim_keys: ['robot0_eef_pos',
'robot0_eef_quat',
'robot0_gripper_qpos',
'object'] # same order of preprocessed observations
multi_step:
n_obs_steps: ${cond_steps}
n_action_steps: ${act_steps}
max_episode_steps: ${env.max_episode_steps}
reset_within_step: True
wandb:
entity: ${oc.env:DPPO_WANDB_ENTITY}
project: rlpd-${env_name}
run: ${now:%H-%M-%S}_${name}
train:
n_train_itr: 1000000
n_steps: 1
gamma: 0.99
actor_lr: 1e-4
actor_weight_decay: 0
actor_lr_scheduler:
first_cycle_steps: 1000
warmup_steps: 10
min_lr: 1e-4
critic_lr: 1e-4
critic_weight_decay: 0
critic_lr_scheduler:
first_cycle_steps: 1000
warmup_steps: 10
min_lr: 1e-4
save_model_freq: 100000
val_freq: 10000
render:
freq: 10000
num: 0
log_freq: 200
# RLPD specific
batch_size: 256
target_ema_rate: 0.01
scale_reward_factor: 1
critic_num_update: 3
buffer_size: 400000
n_eval_episode: 40
n_explore_steps: 0
target_entropy: ${eval:'- ${action_dim} * ${act_steps}'}
init_temperature: 1
model:
_target_: model.rl.gaussian_rlpd.RLPD_Gaussian
randn_clip_value: 10
backup_entropy: True
n_critics: 5
tanh_output: True
actor:
_target_: model.common.mlp_gaussian.Gaussian_MLP
mlp_dims: [512, 512, 512]
activation_type: ReLU
tanh_output: False # squash after sampling instead
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
horizon_steps: ${horizon_steps}
action_dim: ${action_dim}
std_max: 7.3891
std_min: 0.0067
critic:
_target_: model.common.critic.CriticObsAct
mlp_dims: [256, 256, 256]
activation_type: ReLU
use_layernorm: True
double_q: False # use ensemble
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
action_dim: ${action_dim}
action_steps: ${act_steps}
horizon_steps: ${horizon_steps}
device: ${device}
offline_dataset:
_target_: agent.dataset.sequence.StitchedSequenceQLearningDataset
dataset_path: ${offline_dataset_path}
horizon_steps: ${horizon_steps}
cond_steps: ${cond_steps}
device: ${device}
cfg/robomimic/finetune/square/rlpd_mlp.yaml → cfg/robomimic/scratch/square/rlpd_mlp.yaml
+1 -1
View File
@@ -91,7 +91,7 @@ model:
tanh_output: False # squash after sampling instead
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
horizon_steps: ${horizon_steps}
action_dim: ${action_dim}
std_max: 7.3891
std_min: 0.0067
critic:
cfg/robomimic/scratch/square/rlpd_mlp_ph.yaml
+114
View File
@@ -0,0 +1,114 @@
defaults:
- _self_
hydra:
run:
dir: ${logdir}
_target_: agent.finetune.train_rlpd_agent.TrainRLPDAgent
name: ${env_name}_rlpd_mlp_ta${horizon_steps}
logdir: ${oc.env:DPPO_LOG_DIR}/robomimic-finetune/${name}/${now:%Y-%m-%d}_${now:%H-%M-%S}_${seed}
robomimic_env_cfg_path: cfg/robomimic/env_meta/${env_name}.json
normalization_path: ${oc.env:DPPO_DATA_DIR}/robomimic/${env_name}-ph/normalization.npz
offline_dataset_path: ${oc.env:DPPO_DATA_DIR}/robomimic/${env_name}-ph/train.npz
seed: 42
device: cuda:0
env_name: square
obs_dim: 23
action_dim: 7
cond_steps: 1
horizon_steps: 1
act_steps: 1
env:
n_envs: 1
name: ${env_name}
max_episode_steps: 400
reset_at_iteration: False
save_video: False
best_reward_threshold_for_success: 1
wrappers:
robomimic_lowdim:
normalization_path: ${normalization_path}
low_dim_keys: ['robot0_eef_pos',
'robot0_eef_quat',
'robot0_gripper_qpos',
'object'] # same order of preprocessed observations
multi_step:
n_obs_steps: ${cond_steps}
n_action_steps: ${act_steps}
max_episode_steps: ${env.max_episode_steps}
reset_within_step: True
wandb:
entity: ${oc.env:DPPO_WANDB_ENTITY}
project: rlpd-${env_name}
run: ${now:%H-%M-%S}_${name}
train:
n_train_itr: 1000000
n_steps: 1
gamma: 0.99
actor_lr: 1e-4
actor_weight_decay: 0
actor_lr_scheduler:
first_cycle_steps: 1000
warmup_steps: 10
min_lr: 1e-4
critic_lr: 1e-4
critic_weight_decay: 0
critic_lr_scheduler:
first_cycle_steps: 1000
warmup_steps: 10
min_lr: 1e-4
save_model_freq: 100000
val_freq: 10000
render:
freq: 10000
num: 0
log_freq: 200
# RLPD specific
batch_size: 256
target_ema_rate: 0.01
scale_reward_factor: 1
critic_num_update: 3
buffer_size: 400000
n_eval_episode: 40
n_explore_steps: 0
target_entropy: ${eval:'- ${action_dim} * ${act_steps}'}
init_temperature: 1
model:
_target_: model.rl.gaussian_rlpd.RLPD_Gaussian
randn_clip_value: 10
backup_entropy: True
n_critics: 5
tanh_output: True
actor:
_target_: model.common.mlp_gaussian.Gaussian_MLP
mlp_dims: [512, 512, 512]
activation_type: ReLU
tanh_output: False # squash after sampling instead
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
horizon_steps: ${horizon_steps}
action_dim: ${action_dim}
std_max: 7.3891
std_min: 0.0067
critic:
_target_: model.common.critic.CriticObsAct
mlp_dims: [256, 256, 256]
activation_type: ReLU
use_layernorm: True
double_q: False # use ensemble
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
action_dim: ${action_dim}
action_steps: ${act_steps}
horizon_steps: ${horizon_steps}
device: ${device}
offline_dataset:
_target_: agent.dataset.sequence.StitchedSequenceQLearningDataset
dataset_path: ${offline_dataset_path}
horizon_steps: ${horizon_steps}
cond_steps: ${cond_steps}
device: ${device}