dodox/dppo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

v0.7 (#26)

Browse Source 
* update from scratch configs

* update gym pretraining configs - use fewer epochs

* update robomimic pretraining configs - use fewer epochs

* allow trajectory plotting in eval agent

* add simple vit unet

* update avoid pretraining configs - use fewer epochs

* update furniture pretraining configs - use same amount of epochs as before

* add robomimic diffusion unet pretraining configs

* update robomimic finetuning configs - higher lr

* add vit unet checkpoint urls

* update pretraining and finetuning instructions as configs are updated
This commit is contained in:
Allen Z. Ren
2024-11-20 15:56:23 -05:00
committed by allenzren
parent d2929f65e1
commit 1d04211666
158 changed files with 3350 additions and 410 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cfg/robomimic/finetune/can/ft_ppo_diffusion_mlp.yaml
+4 -3
View File
@@ -7,7 +7,8 @@ _target_: agent.finetune.train_ppo_diffusion_agent.TrainPPODiffusionAgent
name: ${env_name}_ft_diffusion_mlp_ta${horizon_steps}_td${denoising_steps}_tdf${ft_denoising_steps}
logdir: ${oc.env:DPPO_LOG_DIR}/robomimic-finetune/${name}/${now:%Y-%m-%d}_${now:%H-%M-%S}_${seed}
base_policy_path: ${oc.env:DPPO_LOG_DIR}/robomimic-pretrain/can/can_pre_diffusion_mlp_ta4_td20/2024-06-28_13-29-54/checkpoint/state_5000.pt # use 8000 for comparing policy parameterizations
base_policy_path: ${oc.env:DPPO_LOG_DIR}/robomimic-pretrain/can/can_pre_diffusion_mlp_ta4_td20/2024-06-28_13-29-54/checkpoint/state_5000.pt # use 5000 for comparing diffusion rl algorithms
# base_policy_path: ${oc.env:DPPO_LOG_DIR}/robomimic-pretrain/can/can_pre_diffusion_mlp_ta4_td20/2024-06-28_13-29-54/checkpoint/state_8000.pt # use 8000 for comparing policy parameterizations
robomimic_env_cfg_path: cfg/robomimic/env_meta/${env_name}.json
normalization_path: ${oc.env:DPPO_DATA_DIR}/robomimic/${env_name}/normalization.npz
@@ -54,13 +55,13 @@ train:
actor_lr: 1e-4
actor_weight_decay: 0
actor_lr_scheduler:
first_cycle_steps: 1000
first_cycle_steps: ${train.n_train_itr}
warmup_steps: 10
min_lr: 1e-4
critic_lr: 1e-3
critic_weight_decay: 0
critic_lr_scheduler:
first_cycle_steps: 1000
first_cycle_steps: ${train.n_train_itr}
warmup_steps: 10
min_lr: 1e-3
save_model_freq: 100
cfg/robomimic/finetune/can/ft_ppo_diffusion_mlp_img.yaml
+4 -4
View File
@@ -66,16 +66,16 @@ train:
gamma: 0.999
augment: True
grad_accumulate: 15
actor_lr: 1e-4
actor_lr: 5e-5
actor_weight_decay: 0
actor_lr_scheduler:
first_cycle_steps: 1000
first_cycle_steps: ${train.n_train_itr}
warmup_steps: 10
min_lr: 1e-4
min_lr: 5e-5
critic_lr: 1e-3
critic_weight_decay: 0
critic_lr_scheduler:
first_cycle_steps: 1000
first_cycle_steps: ${train.n_train_itr}
warmup_steps: 10
min_lr: 1e-3
save_model_freq: 100
cfg/robomimic/finetune/can/ft_ppo_diffusion_unet.yaml
+6 -6
View File
@@ -27,7 +27,7 @@ env:
name: ${env_name}
best_reward_threshold_for_success: 1
max_episode_steps: 300
save_video: false
save_video: False
wrappers:
robomimic_lowdim:
normalization_path: ${normalization_path}
@@ -47,20 +47,20 @@ wandb:
run: ${now:%H-%M-%S}_${name}
train:
n_train_itr: 300
n_train_itr: 151
n_critic_warmup_itr: 2
n_steps: 300
gamma: 0.999
actor_lr: 1e-5
actor_lr: 1e-4
actor_weight_decay: 0
actor_lr_scheduler:
first_cycle_steps: 1000
first_cycle_steps: ${train.n_train_itr}
warmup_steps: 10
min_lr: 1e-5
min_lr: 1e-4
critic_lr: 1e-3
critic_weight_decay: 0
critic_lr_scheduler:
first_cycle_steps: 1000
first_cycle_steps: ${train.n_train_itr}
warmup_steps: 10
min_lr: 1e-3
save_model_freq: 100
cfg/robomimic/finetune/can/ft_ppo_diffusion_unet_img.yaml
+173
View File
@@ -0,0 +1,173 @@
defaults:
- _self_
hydra:
run:
dir: ${logdir}
_target_: agent.finetune.train_ppo_diffusion_img_agent.TrainPPOImgDiffusionAgent
name: ${env_name}_ft_diffusion_unet_img_ta${horizon_steps}_td${denoising_steps}_tdf${ft_denoising_steps}
logdir: ${oc.env:DPPO_LOG_DIR}/robomimic-finetune/${name}/${now:%Y-%m-%d}_${now:%H-%M-%S}_${seed}
base_policy_path: ${oc.env:DPPO_LOG_DIR}/robomimic-pretrain/can/can_pre_diffusion_unet_img_ta4_td100/2024-11-15_17-34-05_42/checkpoint/state_500.pt
robomimic_env_cfg_path: cfg/robomimic/env_meta/${env_name}-img.json
normalization_path: ${oc.env:DPPO_DATA_DIR}/robomimic/${env_name}-img/normalization.npz
seed: 42
device: cuda:0
env_name: can
obs_dim: 9
action_dim: 7
denoising_steps: 100
ft_denoising_steps: 5
cond_steps: 1
img_cond_steps: 1
horizon_steps: 4
act_steps: 4
use_ddim: True
env:
n_envs: 50
name: ${env_name}
best_reward_threshold_for_success: 1
max_episode_steps: 300
save_video: False
use_image_obs: True
wrappers:
robomimic_image:
normalization_path: ${normalization_path}
low_dim_keys: ['robot0_eef_pos',
'robot0_eef_quat',
'robot0_gripper_qpos']
image_keys: ['robot0_eye_in_hand_image']
shape_meta: ${shape_meta}
multi_step:
n_obs_steps: ${cond_steps}
n_action_steps: ${act_steps}
max_episode_steps: ${env.max_episode_steps}
reset_within_step: True
shape_meta:
obs:
rgb:
shape: [3, 96, 96]
state:
shape: [9]
action:
shape: [7]
wandb:
entity: ${oc.env:DPPO_WANDB_ENTITY}
project: robomimic-${env_name}-finetune
run: ${now:%H-%M-%S}_${name}
train:
n_train_itr: 151
n_critic_warmup_itr: 2
n_steps: 300
gamma: 0.999
augment: True
grad_accumulate: 15
actor_lr: 5e-5
actor_weight_decay: 0
actor_lr_scheduler:
first_cycle_steps: ${train.n_train_itr}
warmup_steps: 10
min_lr: 5e-5
critic_lr: 1e-3
critic_weight_decay: 0
critic_lr_scheduler:
first_cycle_steps: ${train.n_train_itr}
warmup_steps: 10
min_lr: 1e-3
save_model_freq: 100
val_freq: 10
render:
freq: 1
num: 0
# PPO specific
reward_scale_running: True
reward_scale_const: 1.0
gae_lambda: 0.95
batch_size: 500
logprob_batch_size: 500
update_epochs: 10
vf_coef: 0.5
target_kl: 1
model:
_target_: model.diffusion.diffusion_ppo.PPODiffusion
# HP to tune
gamma_denoising: 0.99
clip_ploss_coef: 0.01
clip_ploss_coef_base: 0.001
clip_ploss_coef_rate: 3
randn_clip_value: 3
min_sampling_denoising_std: 0.1
min_logprob_denoising_std: 0.1
#
use_ddim: ${use_ddim}
ddim_steps: ${ft_denoising_steps}
learn_eta: False
eta:
base_eta: 1
input_dim: ${obs_dim}
mlp_dims: [256, 256]
action_dim: ${action_dim}
min_eta: 0.1
max_eta: 1.0
_target_: model.diffusion.eta.EtaFixed
network_path: ${base_policy_path}
actor:
_target_: model.diffusion.unet.VisionUnet1D
backbone:
_target_: model.common.vit.VitEncoder
obs_shape: ${shape_meta.obs.rgb.shape}
num_channel: ${eval:'3 * ${img_cond_steps}'} # each image patch is history concatenated
img_h: ${shape_meta.obs.rgb.shape[1]}
img_w: ${shape_meta.obs.rgb.shape[2]}
cfg:
patch_size: 8
depth: 1
embed_dim: 128
num_heads: 4
embed_style: embed2
embed_norm: 0
img_cond_steps: ${img_cond_steps}
augment: False
spatial_emb: 128
diffusion_step_embed_dim: 32
dim: 40
dim_mults: [1, 2]
kernel_size: 5
n_groups: 8
smaller_encoder: False
cond_predict_scale: True
action_dim: ${action_dim}
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
critic:
_target_: model.common.critic.ViTCritic
spatial_emb: 128
augment: False
backbone:
_target_: model.common.vit.VitEncoder
obs_shape: ${shape_meta.obs.rgb.shape}
num_channel: ${eval:'3 * ${img_cond_steps}'} # each image patch is history concatenated
img_h: ${shape_meta.obs.rgb.shape[1]}
img_w: ${shape_meta.obs.rgb.shape[2]}
cfg:
patch_size: 8
depth: 1
embed_dim: 128
num_heads: 4
embed_style: embed2
embed_norm: 0
img_cond_steps: ${img_cond_steps}
mlp_dims: [256, 256, 256]
activation_type: Mish
residual_style: True
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
ft_denoising_steps: ${ft_denoising_steps}
horizon_steps: ${horizon_steps}
obs_dim: ${obs_dim}
action_dim: ${action_dim}
denoising_steps: ${denoising_steps}
device: ${device}
cfg/robomimic/finetune/can/ft_ppo_gaussian_mlp.yaml
+5 -5
View File
@@ -45,20 +45,20 @@ wandb:
run: ${now:%H-%M-%S}_${name}
train:
n_train_itr: 300
n_train_itr: 151
n_critic_warmup_itr: 2
n_steps: 300
gamma: 0.999
actor_lr: 1e-5
actor_lr: 1e-4
actor_weight_decay: 0
actor_lr_scheduler:
first_cycle_steps: 1000
first_cycle_steps: ${train.n_train_itr}
warmup_steps: 10
min_lr: 1e-5
min_lr: 1e-4
critic_lr: 1e-3
critic_weight_decay: 0
critic_lr_scheduler:
first_cycle_steps: 1000
first_cycle_steps: ${train.n_train_itr}
warmup_steps: 10
min_lr: 1e-3
save_model_freq: 100
cfg/robomimic/finetune/can/ft_ppo_gaussian_mlp_img.yaml
+7 -7
View File
@@ -1,7 +1,7 @@
defaults:
- _self_
hydra:
run:
run:
dir: ${logdir}
_target_: agent.finetune.train_ppo_gaussian_img_agent.TrainPPOImgGaussianAgent
@@ -57,22 +57,22 @@ wandb:
run: ${now:%H-%M-%S}_${name}
train:
n_train_itr: 200
n_train_itr: 151
n_critic_warmup_itr: 2
n_steps: 300
gamma: 0.999
augment: True
grad_accumulate: 5
actor_lr: 1e-5
actor_lr: 1e-4
actor_weight_decay: 0
actor_lr_scheduler:
first_cycle_steps: 200
first_cycle_steps: ${train.n_train_itr}
warmup_steps: 10
min_lr: 1e-5
min_lr: 1e-4
critic_lr: 1e-3
critic_weight_decay: 0
critic_lr_scheduler:
first_cycle_steps: 200
first_cycle_steps: ${train.n_train_itr}
warmup_steps: 10
min_lr: 1e-3
save_model_freq: 100
@@ -140,9 +140,9 @@ model:
embed_style: embed2
embed_norm: 0
img_cond_steps: ${img_cond_steps}
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
mlp_dims: [256, 256, 256]
activation_type: Mish
residual_style: True
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
horizon_steps: ${horizon_steps}
device: ${device}
cfg/robomimic/finetune/can/ft_ppo_gaussian_transformer.yaml
+5 -5
View File
@@ -45,20 +45,20 @@ wandb:
run: ${now:%H-%M-%S}_${name}
train:
n_train_itr: 300
n_train_itr: 151
n_critic_warmup_itr: 2
n_steps: 300
gamma: 0.999
actor_lr: 1e-5
actor_lr: 1e-4
actor_weight_decay: 0
actor_lr_scheduler:
first_cycle_steps: 1000
first_cycle_steps: ${train.n_train_itr}
warmup_steps: 10
min_lr: 1e-5
min_lr: 1e-4
critic_lr: 1e-3
critic_weight_decay: 0
critic_lr_scheduler:
first_cycle_steps: 1000
first_cycle_steps: ${train.n_train_itr}
warmup_steps: 10
min_lr: 1e-3
save_model_freq: 100
cfg/robomimic/finetune/can/ft_ppo_gmm_mlp.yaml
+5 -5
View File
@@ -46,20 +46,20 @@ wandb:
run: ${now:%H-%M-%S}_${name}
train:
n_train_itr: 300
n_train_itr: 151
n_critic_warmup_itr: 2
n_steps: 300
gamma: 0.999
actor_lr: 1e-5
actor_lr: 1e-4
actor_weight_decay: 0
actor_lr_scheduler:
first_cycle_steps: 1000
first_cycle_steps: ${train.n_train_itr}
warmup_steps: 10
min_lr: 1e-5
min_lr: 1e-4
critic_lr: 1e-3
critic_weight_decay: 0
critic_lr_scheduler:
first_cycle_steps: 1000
first_cycle_steps: ${train.n_train_itr}
warmup_steps: 10
min_lr: 1e-3
save_model_freq: 100
cfg/robomimic/finetune/lift/ft_ppo_diffusion_mlp.yaml
+5 -4
View File
@@ -1,13 +1,14 @@
defaults:
- _self_
hydra:
run:
run:
dir: ${logdir}
_target_: agent.finetune.train_ppo_diffusion_agent.TrainPPODiffusionAgent
name: ${env_name}_ft_diffusion_mlp_ta${horizon_steps}_td${denoising_steps}_tdf${ft_denoising_steps}
logdir: ${oc.env:DPPO_LOG_DIR}/robomimic-finetune/${name}/${now:%Y-%m-%d}_${now:%H-%M-%S}_${seed}
base_policy_path: ${oc.env:DPPO_LOG_DIR}/robomimic-pretrain/lift/lift_pre_diffusion_mlp_ta4_td20/2024-06-28_14-47-58/checkpoint/state_5000.pt # use 8000 for comparing policy parameterizations
base_policy_path: ${oc.env:DPPO_LOG_DIR}/robomimic-pretrain/lift/lift_pre_diffusion_mlp_ta4_td20/2024-06-28_14-47-58/checkpoint/state_5000.pt # use 5000 for comparing diffusion rl algorithms
# base_policy_path: ${oc.env:DPPO_LOG_DIR}/robomimic-pretrain/lift/lift_pre_diffusion_mlp_ta4_td20/2024-06-28_14-47-58/checkpoint/state_8000.pt # use 8000 for comparing policy parameterizations
robomimic_env_cfg_path: cfg/robomimic/env_meta/${env_name}.json
normalization_path: ${oc.env:DPPO_DATA_DIR}/robomimic/${env_name}/normalization.npz
@@ -54,13 +55,13 @@ train:
actor_lr: 1e-4
actor_weight_decay: 0
actor_lr_scheduler:
first_cycle_steps: 1000
first_cycle_steps: ${train.n_train_itr}
warmup_steps: 10
min_lr: 1e-4
critic_lr: 1e-3
critic_weight_decay: 0
critic_lr_scheduler:
first_cycle_steps: 1000
first_cycle_steps: ${train.n_train_itr}
warmup_steps: 10
min_lr: 1e-3
save_model_freq: 100
cfg/robomimic/finetune/lift/ft_ppo_diffusion_mlp_img.yaml
+5 -5
View File
@@ -60,22 +60,22 @@ wandb:
run: ${now:%H-%M-%S}_${name}
train:
n_train_itr: 151
n_train_itr: 81
n_critic_warmup_itr: 2
n_steps: 300
gamma: 0.999
augment: True
grad_accumulate: 15
actor_lr: 1e-4
actor_lr: 5e-5
actor_weight_decay: 0
actor_lr_scheduler:
first_cycle_steps: 1000
first_cycle_steps: ${train.n_train_itr}
warmup_steps: 10
min_lr: 1e-4
min_lr: 5e-5
critic_lr: 1e-3
critic_weight_decay: 0
critic_lr_scheduler:
first_cycle_steps: 1000
first_cycle_steps: ${train.n_train_itr}
warmup_steps: 10
min_lr: 1e-3
save_model_freq: 100
cfg/robomimic/finetune/lift/ft_ppo_diffusion_unet.yaml
+7 -7
View File
@@ -27,7 +27,7 @@ env:
name: ${env_name}
best_reward_threshold_for_success: 1
max_episode_steps: 300
save_video: false
save_video: False
wrappers:
robomimic_lowdim:
normalization_path: ${normalization_path}
@@ -47,20 +47,20 @@ wandb:
run: ${now:%H-%M-%S}_${name}
train:
n_train_itr: 300
n_train_itr: 81
n_critic_warmup_itr: 2
n_steps: 300
gamma: 0.999
actor_lr: 1e-5
actor_lr: 1e-4
actor_weight_decay: 0
actor_lr_scheduler:
first_cycle_steps: 1000
first_cycle_steps: ${train.n_train_itr}
warmup_steps: 10
min_lr: 1e-5
min_lr: 1e-4
critic_lr: 1e-3
critic_weight_decay: 0
critic_lr_scheduler:
first_cycle_steps: 1000
first_cycle_steps: ${train.n_train_itr}
warmup_steps: 10
min_lr: 1e-3
save_model_freq: 100
@@ -102,10 +102,10 @@ model:
action_dim: ${action_dim}
critic:
_target_: model.common.critic.CriticObs
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
mlp_dims: [256, 256, 256]
activation_type: Mish
residual_style: True
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
ft_denoising_steps: ${ft_denoising_steps}
horizon_steps: ${horizon_steps}
obs_dim: ${obs_dim}
cfg/robomimic/finetune/lift/ft_ppo_diffusion_unet_img.yaml
+173
View File
@@ -0,0 +1,173 @@
defaults:
- _self_
hydra:
run:
dir: ${logdir}
_target_: agent.finetune.train_ppo_diffusion_img_agent.TrainPPOImgDiffusionAgent
name: ${env_name}_ft_diffusion_unet_img_ta${horizon_steps}_td${denoising_steps}_tdf${ft_denoising_steps}
logdir: ${oc.env:DPPO_LOG_DIR}/robomimic-finetune/${name}/${now:%Y-%m-%d}_${now:%H-%M-%S}_${seed}
base_policy_path: ${oc.env:DPPO_LOG_DIR}/robomimic-pretrain/lift/lift_pre_diffusion_unet_img_ta4_td100/2024-11-15_17-35-19_42/checkpoint/state_500.pt
robomimic_env_cfg_path: cfg/robomimic/env_meta/${env_name}-img.json
normalization_path: ${oc.env:DPPO_DATA_DIR}/robomimic/${env_name}-img/normalization.npz
seed: 42
device: cuda:0
env_name: lift
obs_dim: 9
action_dim: 7
denoising_steps: 100
ft_denoising_steps: 5
cond_steps: 1
img_cond_steps: 1
horizon_steps: 4
act_steps: 4
use_ddim: True
env:
n_envs: 50
name: ${env_name}
best_reward_threshold_for_success: 1
max_episode_steps: 300
save_video: False
use_image_obs: True
wrappers:
robomimic_image:
normalization_path: ${normalization_path}
low_dim_keys: ['robot0_eef_pos',
'robot0_eef_quat',
'robot0_gripper_qpos']
image_keys: ['robot0_eye_in_hand_image']
shape_meta: ${shape_meta}
multi_step:
n_obs_steps: ${cond_steps}
n_action_steps: ${act_steps}
max_episode_steps: ${env.max_episode_steps}
reset_within_step: True
shape_meta:
obs:
rgb:
shape: [3, 96, 96]
state:
shape: [9]
action:
shape: [7]
wandb:
entity: ${oc.env:DPPO_WANDB_ENTITY}
project: robomimic-${env_name}-finetune
run: ${now:%H-%M-%S}_${name}
train:
n_train_itr: 81
n_critic_warmup_itr: 2
n_steps: 300
gamma: 0.999
augment: True
grad_accumulate: 15
actor_lr: 5e-5
actor_weight_decay: 0
actor_lr_scheduler:
first_cycle_steps: ${train.n_train_itr}
warmup_steps: 10
min_lr: 5e-5
critic_lr: 1e-3
critic_weight_decay: 0
critic_lr_scheduler:
first_cycle_steps: ${train.n_train_itr}
warmup_steps: 10
min_lr: 1e-3
save_model_freq: 100
val_freq: 10
render:
freq: 1
num: 0
# PPO specific
reward_scale_running: True
reward_scale_const: 1.0
gae_lambda: 0.95
batch_size: 500
logprob_batch_size: 500
update_epochs: 10
vf_coef: 0.5
target_kl: 1
model:
_target_: model.diffusion.diffusion_ppo.PPODiffusion
# HP to tune
gamma_denoising: 0.99
clip_ploss_coef: 0.01
clip_ploss_coef_base: 0.001
clip_ploss_coef_rate: 3
randn_clip_value: 3
min_sampling_denoising_std: 0.1
min_logprob_denoising_std: 0.1
#
use_ddim: ${use_ddim}
ddim_steps: ${ft_denoising_steps}
learn_eta: False
eta:
base_eta: 1
input_dim: ${obs_dim}
mlp_dims: [256, 256]
action_dim: ${action_dim}
min_eta: 0.1
max_eta: 1.0
_target_: model.diffusion.eta.EtaFixed
network_path: ${base_policy_path}
actor:
_target_: model.diffusion.unet.VisionUnet1D
backbone:
_target_: model.common.vit.VitEncoder
obs_shape: ${shape_meta.obs.rgb.shape}
num_channel: ${eval:'3 * ${img_cond_steps}'} # each image patch is history concatenated
img_h: ${shape_meta.obs.rgb.shape[1]}
img_w: ${shape_meta.obs.rgb.shape[2]}
cfg:
patch_size: 8
depth: 1
embed_dim: 128
num_heads: 4
embed_style: embed2
embed_norm: 0
img_cond_steps: ${img_cond_steps}
augment: False
spatial_emb: 128
diffusion_step_embed_dim: 32
dim: 40
dim_mults: [1, 2]
kernel_size: 5
n_groups: 8
smaller_encoder: False
cond_predict_scale: True
action_dim: ${action_dim}
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
critic:
_target_: model.common.critic.ViTCritic
spatial_emb: 128
augment: False
backbone:
_target_: model.common.vit.VitEncoder
obs_shape: ${shape_meta.obs.rgb.shape}
num_channel: ${eval:'3 * ${img_cond_steps}'} # each image patch is history concatenated
img_h: ${shape_meta.obs.rgb.shape[1]}
img_w: ${shape_meta.obs.rgb.shape[2]}
cfg:
patch_size: 8
depth: 1
embed_dim: 128
num_heads: 4
embed_style: embed2
embed_norm: 0
img_cond_steps: ${img_cond_steps}
mlp_dims: [256, 256, 256]
activation_type: Mish
residual_style: True
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
ft_denoising_steps: ${ft_denoising_steps}
horizon_steps: ${horizon_steps}
obs_dim: ${obs_dim}
action_dim: ${action_dim}
denoising_steps: ${denoising_steps}
device: ${device}
cfg/robomimic/finetune/lift/ft_ppo_gaussian_mlp.yaml
+7 -7
View File
@@ -25,7 +25,7 @@ env:
name: ${env_name}
best_reward_threshold_for_success: 1
max_episode_steps: 300
save_video: false
save_video: False
wrappers:
robomimic_lowdim:
normalization_path: ${normalization_path}
@@ -45,20 +45,20 @@ wandb:
run: ${now:%H-%M-%S}_${name}
train:
n_train_itr: 300
n_train_itr: 81
n_critic_warmup_itr: 2
n_steps: 300
gamma: 0.999
actor_lr: 1e-5
actor_lr: 1e-4
actor_weight_decay: 0
actor_lr_scheduler:
first_cycle_steps: 1000
first_cycle_steps: ${train.n_train_itr}
warmup_steps: 10
min_lr: 1e-5
min_lr: 1e-4
critic_lr: 1e-3
critic_weight_decay: 0
critic_lr_scheduler:
first_cycle_steps: 1000
first_cycle_steps: ${train.n_train_itr}
warmup_steps: 10
min_lr: 1e-3
save_model_freq: 100
@@ -93,9 +93,9 @@ model:
action_dim: ${action_dim}
critic:
_target_: model.common.critic.CriticObs
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
mlp_dims: [256, 256, 256]
activation_type: Mish
residual_style: True
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
horizon_steps: ${horizon_steps}
device: ${device}
cfg/robomimic/finetune/lift/ft_ppo_gaussian_mlp_img.yaml
+7 -7
View File
@@ -1,7 +1,7 @@
defaults:
- _self_
hydra:
run:
run:
dir: ${logdir}
_target_: agent.finetune.train_ppo_gaussian_img_agent.TrainPPOImgGaussianAgent
@@ -57,22 +57,22 @@ wandb:
run: ${now:%H-%M-%S}_${name}
train:
n_train_itr: 200
n_train_itr: 81
n_critic_warmup_itr: 2
n_steps: 300
gamma: 0.999
augment: True
grad_accumulate: 5
actor_lr: 1e-5
actor_lr: 1e-4
actor_weight_decay: 0
actor_lr_scheduler:
first_cycle_steps: 200
first_cycle_steps: ${train.n_train_itr}
warmup_steps: 10
min_lr: 1e-5
min_lr: 1e-4
critic_lr: 1e-3
critic_weight_decay: 0
critic_lr_scheduler:
first_cycle_steps: 200
first_cycle_steps: ${train.n_train_itr}
warmup_steps: 10
min_lr: 1e-3
save_model_freq: 100
@@ -140,9 +140,9 @@ model:
embed_style: embed2
embed_norm: 0
img_cond_steps: ${img_cond_steps}
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
mlp_dims: [256, 256, 256]
activation_type: Mish
residual_style: True
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
horizon_steps: ${horizon_steps}
device: ${device}
cfg/robomimic/finetune/lift/ft_ppo_gaussian_transformer.yaml
+7 -7
View File
@@ -25,7 +25,7 @@ env:
name: ${env_name}
best_reward_threshold_for_success: 1
max_episode_steps: 300
save_video: false
save_video: False
wrappers:
robomimic_lowdim:
normalization_path: ${normalization_path}
@@ -45,20 +45,20 @@ wandb:
run: ${now:%H-%M-%S}_${name}
train:
n_train_itr: 300
n_train_itr: 81
n_critic_warmup_itr: 2
n_steps: 300
gamma: 0.999
actor_lr: 1e-5
actor_lr: 1e-4
actor_weight_decay: 0
actor_lr_scheduler:
first_cycle_steps: 1000
first_cycle_steps: ${train.n_train_itr}
warmup_steps: 10
min_lr: 1e-5
min_lr: 1e-4
critic_lr: 1e-3
critic_weight_decay: 0
critic_lr_scheduler:
first_cycle_steps: 1000
first_cycle_steps: ${train.n_train_itr}
warmup_steps: 10
min_lr: 1e-3
save_model_freq: 100
@@ -94,9 +94,9 @@ model:
action_dim: ${action_dim}
critic:
_target_: model.common.critic.CriticObs
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
mlp_dims: [256, 256, 256]
activation_type: Mish
residual_style: True
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
horizon_steps: ${horizon_steps}
device: ${device}
cfg/robomimic/finetune/lift/ft_ppo_gmm_mlp.yaml
+7 -7
View File
@@ -26,7 +26,7 @@ env:
name: ${env_name}
best_reward_threshold_for_success: 1
max_episode_steps: 300
save_video: false
save_video: False
wrappers:
robomimic_lowdim:
normalization_path: ${normalization_path}
@@ -46,20 +46,20 @@ wandb:
run: ${now:%H-%M-%S}_${name}
train:
n_train_itr: 300
n_train_itr: 81
n_critic_warmup_itr: 2
n_steps: 300
gamma: 0.999
actor_lr: 1e-5
actor_lr: 1e-4
actor_weight_decay: 0
actor_lr_scheduler:
first_cycle_steps: 1000
first_cycle_steps: ${train.n_train_itr}
warmup_steps: 10
min_lr: 1e-5
min_lr: 1e-4
critic_lr: 1e-3
critic_weight_decay: 0
critic_lr_scheduler:
first_cycle_steps: 1000
first_cycle_steps: ${train.n_train_itr}
warmup_steps: 10
min_lr: 1e-3
save_model_freq: 100
@@ -94,9 +94,9 @@ model:
action_dim: ${action_dim}
critic:
_target_: model.common.critic.CriticObs
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
mlp_dims: [256, 256, 256]
activation_type: Mish
residual_style: True
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
horizon_steps: ${horizon_steps}
device: ${device}
cfg/robomimic/finetune/lift/ft_ppo_gmm_transformer.yaml
+7 -7
View File
@@ -26,7 +26,7 @@ env:
name: ${env_name}
best_reward_threshold_for_success: 1
max_episode_steps: 300
save_video: false
save_video: False
wrappers:
robomimic_lowdim:
normalization_path: ${normalization_path}
@@ -46,20 +46,20 @@ wandb:
run: ${now:%H-%M-%S}_${name}
train:
n_train_itr: 300
n_train_itr: 81
n_critic_warmup_itr: 2
n_steps: 300
gamma: 0.999
actor_lr: 1e-5
actor_lr: 1e-4
actor_weight_decay: 0
actor_lr_scheduler:
first_cycle_steps: 1000
first_cycle_steps: ${train.n_train_itr}
warmup_steps: 10
min_lr: 1e-5
min_lr: 1e-4
critic_lr: 1e-3
critic_weight_decay: 0
critic_lr_scheduler:
first_cycle_steps: 1000
first_cycle_steps: ${train.n_train_itr}
warmup_steps: 10
min_lr: 1e-3
save_model_freq: 100
@@ -95,9 +95,9 @@ model:
action_dim: ${action_dim}
critic:
_target_: model.common.critic.CriticObs
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
mlp_dims: [256, 256, 256]
activation_type: Mish
residual_style: True
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
horizon_steps: ${horizon_steps}
device: ${device}
cfg/robomimic/finetune/square/ft_ppo_diffusion_mlp.yaml
+6 -6
View File
@@ -1,7 +1,7 @@
defaults:
- _self_
hydra:
run:
run:
dir: ${logdir}
_target_: agent.finetune.train_ppo_diffusion_agent.TrainPPODiffusionAgent
@@ -27,7 +27,7 @@ env:
name: ${env_name}
best_reward_threshold_for_success: 1
max_episode_steps: 400
save_video: false
save_video: False
wrappers:
robomimic_lowdim:
normalization_path: ${normalization_path}
@@ -54,14 +54,14 @@ train:
actor_lr: 1e-4
actor_weight_decay: 0
actor_lr_scheduler:
first_cycle_steps: 1000
warmup_steps: 10
first_cycle_steps: ${train.n_train_itr}
warmup_steps: 0
min_lr: 1e-4
critic_lr: 1e-3
critic_weight_decay: 0
critic_lr_scheduler:
first_cycle_steps: 1000
warmup_steps: 10
first_cycle_steps: ${train.n_train_itr}
warmup_steps: 0
min_lr: 1e-3
save_model_freq: 100
val_freq: 10
cfg/robomimic/finetune/square/ft_ppo_diffusion_mlp_img.yaml
+2 -2
View File
@@ -69,13 +69,13 @@ train:
actor_lr: 1e-5
actor_weight_decay: 0
actor_lr_scheduler:
first_cycle_steps: 1000
first_cycle_steps: ${train.n_train_itr}
warmup_steps: 10
min_lr: 1e-5
critic_lr: 1e-3
critic_weight_decay: 0
critic_lr_scheduler:
first_cycle_steps: 1000
first_cycle_steps: ${train.n_train_itr}
warmup_steps: 10
min_lr: 1e-3
save_model_freq: 100
cfg/robomimic/finetune/square/ft_ppo_diffusion_unet.yaml
+10 -10
View File
@@ -1,7 +1,7 @@
defaults:
- _self_
hydra:
run:
run:
dir: ${logdir}
_target_: agent.finetune.train_ppo_diffusion_agent.TrainPPODiffusionAgent
@@ -27,7 +27,7 @@ env:
name: ${env_name}
best_reward_threshold_for_success: 1
max_episode_steps: 400
save_video: false
save_video: False
wrappers:
robomimic_lowdim:
normalization_path: ${normalization_path}
@@ -47,21 +47,21 @@ wandb:
run: ${now:%H-%M-%S}_${name}
train:
n_train_itr: 1000
n_train_itr: 201
n_critic_warmup_itr: 2
n_steps: 400
gamma: 0.999
actor_lr: 1e-5
actor_lr: 2e-5
actor_weight_decay: 0
actor_lr_scheduler:
first_cycle_steps: 1000
warmup_steps: 10
min_lr: 1e-5
first_cycle_steps: ${train.n_train_itr}
warmup_steps: 0
min_lr: 1e-4
critic_lr: 1e-3
critic_weight_decay: 0
critic_lr_scheduler:
first_cycle_steps: 1000
warmup_steps: 10
first_cycle_steps: ${train.n_train_itr}
warmup_steps: 0
min_lr: 1e-3
save_model_freq: 100
val_freq: 10
@@ -102,10 +102,10 @@ model:
action_dim: ${action_dim}
critic:
_target_: model.common.critic.CriticObs
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
mlp_dims: [256, 256, 256]
activation_type: Mish
residual_style: True
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
ft_denoising_steps: ${ft_denoising_steps}
horizon_steps: ${horizon_steps}
obs_dim: ${obs_dim}
cfg/robomimic/finetune/square/ft_ppo_diffusion_unet_img.yaml
+173
View File
@@ -0,0 +1,173 @@
defaults:
- _self_
hydra:
run:
dir: ${logdir}
_target_: agent.finetune.train_ppo_diffusion_img_agent.TrainPPOImgDiffusionAgent
name: ${env_name}_ft_diffusion_unet_img_ta${horizon_steps}_td${denoising_steps}_tdf${ft_denoising_steps}
logdir: ${oc.env:DPPO_LOG_DIR}/robomimic-finetune/${name}/${now:%Y-%m-%d}_${now:%H-%M-%S}_${seed}
base_policy_path: ${oc.env:DPPO_LOG_DIR}/robomimic-pretrain/square/square_pre_diffusion_unet_img_ta4_td100/2024-11-15_17-36-37_42/checkpoint/state_500.pt
robomimic_env_cfg_path: cfg/robomimic/env_meta/${env_name}-img.json
normalization_path: ${oc.env:DPPO_DATA_DIR}/robomimic/${env_name}-img/normalization.npz
seed: 42
device: cuda:0
env_name: square
obs_dim: 9
action_dim: 7
denoising_steps: 100
ft_denoising_steps: 5
cond_steps: 1
img_cond_steps: 1
horizon_steps: 4
act_steps: 4
use_ddim: True
env:
n_envs: 50
name: ${env_name}
best_reward_threshold_for_success: 1
max_episode_steps: 400
save_video: False
use_image_obs: True
wrappers:
robomimic_image:
normalization_path: ${normalization_path}
low_dim_keys: ['robot0_eef_pos',
'robot0_eef_quat',
'robot0_gripper_qpos']
image_keys: ['agentview_image']
shape_meta: ${shape_meta}
multi_step:
n_obs_steps: ${cond_steps}
n_action_steps: ${act_steps}
max_episode_steps: ${env.max_episode_steps}
reset_within_step: True
shape_meta:
obs:
rgb:
shape: [3, 96, 96]
state:
shape: [9]
action:
shape: [7]
wandb:
entity: ${oc.env:DPPO_WANDB_ENTITY}
project: robomimic-${env_name}-finetune
run: ${now:%H-%M-%S}_${name}
train:
n_train_itr: 301
n_critic_warmup_itr: 2
n_steps: 400
gamma: 0.999
augment: True
grad_accumulate: 20
actor_lr: 1e-5
actor_weight_decay: 0
actor_lr_scheduler:
first_cycle_steps: ${train.n_train_itr}
warmup_steps: 10
min_lr: 1e-5
critic_lr: 1e-3
critic_weight_decay: 0
critic_lr_scheduler:
first_cycle_steps: ${train.n_train_itr}
warmup_steps: 10
min_lr: 1e-3
save_model_freq: 100
val_freq: 10
render:
freq: 1
num: 0
# PPO specific
reward_scale_running: True
reward_scale_const: 1.0
gae_lambda: 0.95
batch_size: 500
logprob_batch_size: 1000
update_epochs: 10
vf_coef: 0.5
target_kl: 1
model:
_target_: model.diffusion.diffusion_ppo.PPODiffusion
# HP to tune
gamma_denoising: 0.99
clip_ploss_coef: 0.01
clip_ploss_coef_base: 0.001
clip_ploss_coef_rate: 3
randn_clip_value: 3
min_sampling_denoising_std: 0.1
min_logprob_denoising_std: 0.1
#
use_ddim: ${use_ddim}
ddim_steps: ${ft_denoising_steps}
learn_eta: False
eta:
base_eta: 1
input_dim: ${obs_dim}
mlp_dims: [256, 256]
action_dim: ${action_dim}
min_eta: 0.1
max_eta: 1.0
_target_: model.diffusion.eta.EtaFixed
network_path: ${base_policy_path}
actor:
_target_: model.diffusion.unet.VisionUnet1D
backbone:
_target_: model.common.vit.VitEncoder
obs_shape: ${shape_meta.obs.rgb.shape}
num_channel: ${eval:'3 * ${img_cond_steps}'} # each image patch is history concatenated
img_h: ${shape_meta.obs.rgb.shape[1]}
img_w: ${shape_meta.obs.rgb.shape[2]}
cfg:
patch_size: 8
depth: 1
embed_dim: 128
num_heads: 4
embed_style: embed2
embed_norm: 0
img_cond_steps: ${img_cond_steps}
augment: False
spatial_emb: 128
diffusion_step_embed_dim: 32
dim: 64
dim_mults: [1, 2]
kernel_size: 5
n_groups: 8
smaller_encoder: False
cond_predict_scale: True
action_dim: ${action_dim}
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
critic:
_target_: model.common.critic.ViTCritic
spatial_emb: 128
augment: False
backbone:
_target_: model.common.vit.VitEncoder
obs_shape: ${shape_meta.obs.rgb.shape}
num_channel: ${eval:'3 * ${img_cond_steps}'} # each image patch is history concatenated
img_h: ${shape_meta.obs.rgb.shape[1]}
img_w: ${shape_meta.obs.rgb.shape[2]}
cfg:
patch_size: 8
depth: 1
embed_dim: 128
num_heads: 4
embed_style: embed2
embed_norm: 0
img_cond_steps: ${img_cond_steps}
mlp_dims: [256, 256, 256]
activation_type: Mish
residual_style: True
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
ft_denoising_steps: ${ft_denoising_steps}
horizon_steps: ${horizon_steps}
obs_dim: ${obs_dim}
action_dim: ${action_dim}
denoising_steps: ${denoising_steps}
device: ${device}
cfg/robomimic/finetune/square/ft_ppo_gaussian_mlp.yaml
+9 -9
View File
@@ -25,7 +25,7 @@ env:
name: ${env_name}
best_reward_threshold_for_success: 1
max_episode_steps: 400
save_video: false
save_video: False
wrappers:
robomimic_lowdim:
normalization_path: ${normalization_path}
@@ -45,21 +45,21 @@ wandb:
run: ${now:%H-%M-%S}_${name}
train:
n_train_itr: 1000
n_train_itr: 201
n_critic_warmup_itr: 2
n_steps: 400
gamma: 0.999
actor_lr: 1e-5
actor_lr: 1e-4
actor_weight_decay: 0
actor_lr_scheduler:
first_cycle_steps: 1000
warmup_steps: 10
min_lr: 1e-5
first_cycle_steps: ${train.n_train_itr}
warmup_steps: 0
min_lr: 1e-4
critic_lr: 1e-3
critic_weight_decay: 0
critic_lr_scheduler:
first_cycle_steps: 1000
warmup_steps: 10
first_cycle_steps: ${train.n_train_itr}
warmup_steps: 0
min_lr: 1e-3
save_model_freq: 100
val_freq: 10
@@ -93,9 +93,9 @@ model:
action_dim: ${action_dim}
critic:
_target_: model.common.critic.CriticObs
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
mlp_dims: [256, 256, 256]
activation_type: Mish
residual_style: True
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
horizon_steps: ${horizon_steps}
device: ${device}
cfg/robomimic/finetune/square/ft_ppo_gaussian_mlp_img.yaml
+4 -4
View File
@@ -57,7 +57,7 @@ wandb:
run: ${now:%H-%M-%S}_${name}
train:
n_train_itr: 500
n_train_itr: 301
n_critic_warmup_itr: 2
n_steps: 400
gamma: 0.999
@@ -66,13 +66,13 @@ train:
actor_lr: 1e-5
actor_weight_decay: 0
actor_lr_scheduler:
first_cycle_steps: 500
first_cycle_steps: ${train.n_train_itr}
warmup_steps: 10
min_lr: 1e-5
critic_lr: 1e-3
critic_weight_decay: 0
critic_lr_scheduler:
first_cycle_steps: 500
first_cycle_steps: ${train.n_train_itr}
warmup_steps: 10
min_lr: 1e-3
save_model_freq: 100
@@ -140,9 +140,9 @@ model:
embed_style: embed2
embed_norm: 0
img_cond_steps: ${img_cond_steps}
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
mlp_dims: [256, 256, 256]
activation_type: Mish
residual_style: True
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
horizon_steps: ${horizon_steps}
device: ${device}
cfg/robomimic/finetune/square/ft_ppo_gaussian_transformer.yaml
+9 -9
View File
@@ -25,7 +25,7 @@ env:
name: ${env_name}
best_reward_threshold_for_success: 1
max_episode_steps: 400
save_video: false
save_video: False
wrappers:
robomimic_lowdim:
normalization_path: ${normalization_path}
@@ -45,21 +45,21 @@ wandb:
run: ${now:%H-%M-%S}_${name}
train:
n_train_itr: 1000
n_train_itr: 201
n_critic_warmup_itr: 2
n_steps: 400
gamma: 0.999
actor_lr: 1e-5
actor_lr: 1e-4
actor_weight_decay: 0
actor_lr_scheduler:
first_cycle_steps: 1000
warmup_steps: 10
min_lr: 1e-5
first_cycle_steps: ${train.n_train_itr}
warmup_steps: 0
min_lr: 1e-4
critic_lr: 1e-3
critic_weight_decay: 0
critic_lr_scheduler:
first_cycle_steps: 1000
warmup_steps: 10
first_cycle_steps: ${train.n_train_itr}
warmup_steps: 0
min_lr: 1e-3
save_model_freq: 100
val_freq: 10
@@ -94,9 +94,9 @@ model:
action_dim: ${action_dim}
critic:
_target_: model.common.critic.CriticObs
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
mlp_dims: [256, 256, 256]
activation_type: Mish
residual_style: True
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
horizon_steps: ${horizon_steps}
device: ${device}
cfg/robomimic/finetune/square/ft_ppo_gmm_mlp.yaml
+9 -9
View File
@@ -26,7 +26,7 @@ env:
name: ${env_name}
best_reward_threshold_for_success: 1
max_episode_steps: 400
save_video: false
save_video: False
wrappers:
robomimic_lowdim:
normalization_path: ${normalization_path}
@@ -46,21 +46,21 @@ wandb:
run: ${now:%H-%M-%S}_${name}
train:
n_train_itr: 1000
n_train_itr: 201
n_critic_warmup_itr: 2
n_steps: 400
gamma: 0.999
actor_lr: 1e-5
actor_lr: 1e-4
actor_weight_decay: 0
actor_lr_scheduler:
first_cycle_steps: 1000
warmup_steps: 10
min_lr: 1e-5
first_cycle_steps: ${train.n_train_itr}
warmup_steps: 0
min_lr: 1e-4
critic_lr: 1e-3
critic_weight_decay: 0
critic_lr_scheduler:
first_cycle_steps: 1000
warmup_steps: 10
first_cycle_steps: ${train.n_train_itr}
warmup_steps: 0
min_lr: 1e-3
save_model_freq: 100
val_freq: 10
@@ -94,9 +94,9 @@ model:
action_dim: ${action_dim}
critic:
_target_: model.common.critic.CriticObs
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
mlp_dims: [256, 256, 256]
activation_type: Mish
residual_style: True
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
horizon_steps: ${horizon_steps}
device: ${device}
cfg/robomimic/finetune/square/ft_ppo_gmm_transformer.yaml
+9 -9
View File
@@ -26,7 +26,7 @@ env:
name: ${env_name}
best_reward_threshold_for_success: 1
max_episode_steps: 400
save_video: false
save_video: False
wrappers:
robomimic_lowdim:
normalization_path: ${normalization_path}
@@ -46,21 +46,21 @@ wandb:
run: ${now:%H-%M-%S}_${name}
train:
n_train_itr: 1000
n_train_itr: 201
n_critic_warmup_itr: 2
n_steps: 400
gamma: 0.999
actor_lr: 1e-5
actor_lr: 1e-4
actor_weight_decay: 0
actor_lr_scheduler:
first_cycle_steps: 1000
warmup_steps: 10
min_lr: 1e-5
first_cycle_steps: ${train.n_train_itr}
warmup_steps: 0
min_lr: 1e-4
critic_lr: 1e-3
critic_weight_decay: 0
critic_lr_scheduler:
first_cycle_steps: 1000
warmup_steps: 10
first_cycle_steps: ${train.n_train_itr}
warmup_steps: 0
min_lr: 1e-3
save_model_freq: 100
val_freq: 10
@@ -95,9 +95,9 @@ model:
action_dim: ${action_dim}
critic:
_target_: model.common.critic.CriticObs
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
mlp_dims: [256, 256, 256]
activation_type: Mish
residual_style: True
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
horizon_steps: ${horizon_steps}
device: ${device}
cfg/robomimic/finetune/transport/ft_ppo_diffusion_mlp.yaml
+3 -3
View File
@@ -27,7 +27,7 @@ env:
name: ${env_name}
best_reward_threshold_for_success: 1
max_episode_steps: 800
save_video: false
save_video: False
wrappers:
robomimic_lowdim:
normalization_path: ${normalization_path}
@@ -57,13 +57,13 @@ train:
actor_lr: 1e-4
actor_weight_decay: 0
actor_lr_scheduler:
first_cycle_steps: 1000
first_cycle_steps: ${train.n_train_itr}
warmup_steps: 10
min_lr: 1e-4
critic_lr: 1e-3
critic_weight_decay: 0
critic_lr_scheduler:
first_cycle_steps: 1000
first_cycle_steps: ${train.n_train_itr}
warmup_steps: 10
min_lr: 1e-3
save_model_freq: 100
cfg/robomimic/finetune/transport/ft_ppo_diffusion_mlp_img.yaml
+2 -2
View File
@@ -73,13 +73,13 @@ train:
actor_lr: 1e-5
actor_weight_decay: 0
actor_lr_scheduler:
first_cycle_steps: 1000
first_cycle_steps: ${train.n_train_itr}
warmup_steps: 10
min_lr: 1e-5
critic_lr: 1e-3
critic_weight_decay: 0
critic_lr_scheduler:
first_cycle_steps: 1000
first_cycle_steps: ${train.n_train_itr}
warmup_steps: 10
min_lr: 1e-3
save_model_freq: 100
cfg/robomimic/finetune/transport/ft_ppo_diffusion_unet.yaml
+11 -11
View File
@@ -27,7 +27,7 @@ env:
name: ${env_name}
best_reward_threshold_for_success: 1
max_episode_steps: 800
save_video: false
save_video: False
wrappers:
robomimic_lowdim:
normalization_path: ${normalization_path}
@@ -50,20 +50,20 @@ wandb:
run: ${now:%H-%M-%S}_${name}
train:
n_train_itr: 1000
n_train_itr: 201
n_critic_warmup_itr: 2
n_steps: 400
gamma: 0.999
actor_lr: 1e-5
actor_lr: 2e-5
actor_weight_decay: 0
actor_lr_scheduler:
first_cycle_steps: 1000
first_cycle_steps: ${train.n_train_itr}
warmup_steps: 10
min_lr: 1e-6
min_lr: 1e-4
critic_lr: 1e-3
critic_weight_decay: 0
critic_lr_scheduler:
first_cycle_steps: 1000
first_cycle_steps: ${train.n_train_itr}
warmup_steps: 10
min_lr: 1e-3
save_model_freq: 100
@@ -76,7 +76,7 @@ train:
reward_scale_const: 1.0
gae_lambda: 0.95
batch_size: 10000
update_epochs: 8
update_epochs: 5
vf_coef: 0.5
target_kl: 1
@@ -84,11 +84,11 @@ model:
_target_: model.diffusion.diffusion_ppo.PPODiffusion
# HP to tune
gamma_denoising: 0.99
clip_ploss_coef: 0.001
clip_ploss_coef_base: 0.0001
clip_ploss_coef: 0.01
clip_ploss_coef_base: 0.001
clip_ploss_coef_rate: 3
randn_clip_value: 3
min_sampling_denoising_std: 0.08
min_sampling_denoising_std: 0.1
min_logprob_denoising_std: 0.1
#
network_path: ${base_policy_path}
@@ -105,10 +105,10 @@ model:
action_dim: ${action_dim}
critic:
_target_: model.common.critic.CriticObs
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
mlp_dims: [256, 256, 256]
activation_type: Mish
residual_style: True
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
ft_denoising_steps: ${ft_denoising_steps}
horizon_steps: ${horizon_steps}
obs_dim: ${obs_dim}
cfg/robomimic/finetune/transport/ft_ppo_diffusion_unet_img.yaml
+179
View File
@@ -0,0 +1,179 @@
defaults:
- _self_
hydra:
run:
dir: ${logdir}
_target_: agent.finetune.train_ppo_diffusion_img_agent.TrainPPOImgDiffusionAgent
name: ${env_name}_ft_diffusion_unet_img_ta${horizon_steps}_td${denoising_steps}_tdf${ft_denoising_steps}
logdir: ${oc.env:DPPO_LOG_DIR}/robomimic-finetune/${name}/${now:%Y-%m-%d}_${now:%H-%M-%S}_${seed}
base_policy_path: ${oc.env:DPPO_LOG_DIR}/robomimic-pretrain/transport/transport_pre_diffusion_unet_img_ta16_td100/2024-11-15_17-55-22_42/checkpoint/state_1000.pt
robomimic_env_cfg_path: cfg/robomimic/env_meta/${env_name}-img.json
normalization_path: ${oc.env:DPPO_DATA_DIR}/robomimic/${env_name}-img/normalization.npz
seed: 42
device: cuda:0
env_name: transport
obs_dim: 18
action_dim: 14
denoising_steps: 100
ft_denoising_steps: 5
cond_steps: 1
img_cond_steps: 1
horizon_steps: 16
act_steps: 8
use_ddim: True
env:
n_envs: 50
name: ${env_name}
best_reward_threshold_for_success: 1
max_episode_steps: 800
save_video: False
use_image_obs: True
wrappers:
robomimic_image:
normalization_path: ${normalization_path}
low_dim_keys: ['robot0_eef_pos',
'robot0_eef_quat',
'robot0_gripper_qpos',
"robot1_eef_pos",
"robot1_eef_quat",
"robot1_gripper_qpos"]
image_keys: ['shouldercamera0_image',
'shouldercamera1_image']
shape_meta: ${shape_meta}
multi_step:
n_obs_steps: ${cond_steps}
n_action_steps: ${act_steps}
max_episode_steps: ${env.max_episode_steps}
reset_within_step: True
shape_meta:
obs:
rgb:
shape: [6, 96, 96]
state:
shape: [18]
action:
shape: [14]
wandb:
entity: ${oc.env:DPPO_WANDB_ENTITY}
project: robomimic-${env_name}-finetune
run: ${now:%H-%M-%S}_${name}
train:
n_train_itr: 201
n_critic_warmup_itr: 2
n_steps: 400
gamma: 0.999
augment: True
grad_accumulate: 20
actor_lr: 2e-5
actor_weight_decay: 0
actor_lr_scheduler:
first_cycle_steps: ${train.n_train_itr}
warmup_steps: 10
min_lr: 2e-5
critic_lr: 1e-3
critic_weight_decay: 0
critic_lr_scheduler:
first_cycle_steps: ${train.n_train_itr}
warmup_steps: 10
min_lr: 1e-3
save_model_freq: 100
val_freq: 10
render:
freq: 1
num: 0
# PPO specific
reward_scale_running: True
reward_scale_const: 1.0
gae_lambda: 0.95
batch_size: 500
logprob_batch_size: 1000
update_epochs: 10
vf_coef: 0.5
target_kl: 1
model:
_target_: model.diffusion.diffusion_ppo.PPODiffusion
# HP to tune
gamma_denoising: 0.99
clip_ploss_coef: 0.01
clip_ploss_coef_base: 0.001
clip_ploss_coef_rate: 3
randn_clip_value: 3
min_sampling_denoising_std: 0.1
min_logprob_denoising_std: 0.1
#
use_ddim: ${use_ddim}
ddim_steps: ${ft_denoising_steps}
learn_eta: False
eta:
base_eta: 1
input_dim: ${obs_dim}
mlp_dims: [256, 256]
action_dim: ${action_dim}
min_eta: 0.1
max_eta: 1.0
_target_: model.diffusion.eta.EtaFixed
network_path: ${base_policy_path}
actor:
_target_: model.diffusion.unet.VisionUnet1D
backbone:
_target_: model.common.vit.VitEncoder
obs_shape: ${shape_meta.obs.rgb.shape}
num_channel: ${eval:'3 * ${img_cond_steps}'} # each image patch is history concatenated
img_h: ${shape_meta.obs.rgb.shape[1]}
img_w: ${shape_meta.obs.rgb.shape[2]}
cfg:
patch_size: 8
depth: 1
embed_dim: 128
num_heads: 4
embed_style: embed2
embed_norm: 0
img_cond_steps: ${img_cond_steps}
augment: False
num_img: 2
spatial_emb: 128
diffusion_step_embed_dim: 32
dim: 64
dim_mults: [1, 2]
kernel_size: 5
n_groups: 8
smaller_encoder: False
cond_predict_scale: True
action_dim: ${action_dim}
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
critic:
_target_: model.common.critic.ViTCritic
spatial_emb: 128
num_img: 2
augment: False
backbone:
_target_: model.common.vit.VitEncoder
obs_shape: ${shape_meta.obs.rgb.shape}
num_channel: ${eval:'3 * ${img_cond_steps}'} # each image patch is history concatenated
img_h: ${shape_meta.obs.rgb.shape[1]}
img_w: ${shape_meta.obs.rgb.shape[2]}
cfg:
patch_size: 8
depth: 1
embed_dim: 128
num_heads: 4
embed_style: embed2
embed_norm: 0
img_cond_steps: ${img_cond_steps}
mlp_dims: [256, 256, 256]
activation_type: Mish
residual_style: True
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
ft_denoising_steps: ${ft_denoising_steps}
horizon_steps: ${horizon_steps}
obs_dim: ${obs_dim}
action_dim: ${action_dim}
denoising_steps: ${denoising_steps}
device: ${device}
cfg/robomimic/finetune/transport/ft_ppo_gaussian_mlp.yaml
+11 -11
View File
@@ -25,7 +25,7 @@ env:
name: ${env_name}
best_reward_threshold_for_success: 1
max_episode_steps: 800
save_video: false
save_video: False
wrappers:
robomimic_lowdim:
normalization_path: ${normalization_path}
@@ -48,21 +48,21 @@ wandb:
run: ${now:%H-%M-%S}_${name}
train:
n_train_itr: 1000
n_train_itr: 201
n_critic_warmup_itr: 2
n_steps: 400
gamma: 0.999
actor_lr: 1e-5
actor_lr: 1e-4
actor_weight_decay: 0
actor_lr_scheduler:
first_cycle_steps: 1000
warmup_steps: 10
min_lr: 1e-6
first_cycle_steps: ${train.n_train_itr}
warmup_steps: 0
min_lr: 1e-4
critic_lr: 1e-3
critic_weight_decay: 0
critic_lr_scheduler:
first_cycle_steps: 1000
warmup_steps: 10
first_cycle_steps: ${train.n_train_itr}
warmup_steps: 0
min_lr: 1e-3
save_model_freq: 100
val_freq: 10
@@ -74,7 +74,7 @@ train:
reward_scale_const: 1.0
gae_lambda: 0.95
batch_size: 10000
update_epochs: 8
update_epochs: 5
vf_coef: 0.5
target_kl: 1
@@ -87,7 +87,7 @@ model:
_target_: model.common.mlp_gaussian.Gaussian_MLP
mlp_dims: [1024, 1024, 1024]
residual_style: True
fixed_std: 0.08
fixed_std: 0.1
learn_fixed_std: True
std_min: 0.01
std_max: 0.2
@@ -96,9 +96,9 @@ model:
action_dim: ${action_dim}
critic:
_target_: model.common.critic.CriticObs
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
mlp_dims: [256, 256, 256]
activation_type: Mish
residual_style: True
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
horizon_steps: ${horizon_steps}
device: ${device}
cfg/robomimic/finetune/transport/ft_ppo_gaussian_mlp_img.yaml
+7 -7
View File
@@ -61,7 +61,7 @@ wandb:
run: ${now:%H-%M-%S}_${name}
train:
n_train_itr: 500
n_train_itr: 201
n_critic_warmup_itr: 2
n_steps: 400
gamma: 0.999
@@ -70,13 +70,13 @@ train:
actor_lr: 1e-5
actor_weight_decay: 0
actor_lr_scheduler:
first_cycle_steps: 500
first_cycle_steps: ${train.n_train_itr}
warmup_steps: 10
min_lr: 1e-6
min_lr: 1e-5
critic_lr: 1e-3
critic_weight_decay: 0
critic_lr_scheduler:
first_cycle_steps: 500
first_cycle_steps: ${train.n_train_itr}
warmup_steps: 10
min_lr: 1e-3
save_model_freq: 100
@@ -90,7 +90,7 @@ train:
gae_lambda: 0.95
batch_size: 1000
logprob_batch_size: 1000
update_epochs: 8
update_epochs: 10
vf_coef: 0.5
target_kl: 1
@@ -119,7 +119,7 @@ model:
spatial_emb: 128
mlp_dims: [768, 768, 768]
residual_style: True
fixed_std: 0.08
fixed_std: 0.1
learn_fixed_std: True
std_min: 0.01
std_max: 0.2
@@ -146,9 +146,9 @@ model:
embed_style: embed2
embed_norm: 0
img_cond_steps: ${img_cond_steps}
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
mlp_dims: [256, 256, 256]
activation_type: Mish
residual_style: True
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
horizon_steps: ${horizon_steps}
device: ${device}
cfg/robomimic/finetune/transport/ft_ppo_gaussian_transformer.yaml
+8 -8
View File
@@ -25,7 +25,7 @@ env:
name: ${env_name}
best_reward_threshold_for_success: 1
max_episode_steps: 800
save_video: false
save_video: False
wrappers:
robomimic_lowdim:
normalization_path: ${normalization_path}
@@ -48,20 +48,20 @@ wandb:
run: ${now:%H-%M-%S}_${name}
train:
n_train_itr: 1000
n_train_itr: 201
n_critic_warmup_itr: 2
n_steps: 400
gamma: 0.999
actor_lr: 1e-5
actor_lr: 1e-4
actor_weight_decay: 0
actor_lr_scheduler:
first_cycle_steps: 1000
first_cycle_steps: ${train.n_train_itr}
warmup_steps: 10
min_lr: 1e-6
min_lr: 1e-4
critic_lr: 1e-3
critic_weight_decay: 0
critic_lr_scheduler:
first_cycle_steps: 1000
first_cycle_steps: ${train.n_train_itr}
warmup_steps: 10
min_lr: 1e-3
save_model_freq: 100
@@ -74,7 +74,7 @@ train:
reward_scale_const: 1.0
gae_lambda: 0.95
batch_size: 10000
update_epochs: 8
update_epochs: 5
vf_coef: 0.5
target_kl: 1
@@ -97,9 +97,9 @@ model:
action_dim: ${action_dim}
critic:
_target_: model.common.critic.CriticObs
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
mlp_dims: [256, 256, 256]
activation_type: Mish
residual_style: True
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
horizon_steps: ${horizon_steps}
device: ${device}
cfg/robomimic/finetune/transport/ft_ppo_gmm_mlp.yaml
+11 -11
View File
@@ -26,7 +26,7 @@ env:
name: ${env_name}
best_reward_threshold_for_success: 1
max_episode_steps: 800
save_video: false
save_video: False
wrappers:
robomimic_lowdim:
normalization_path: ${normalization_path}
@@ -49,21 +49,21 @@ wandb:
run: ${now:%H-%M-%S}_${name}
train:
n_train_itr: 1000
n_train_itr: 201
n_critic_warmup_itr: 2
n_steps: 400
gamma: 0.999
actor_lr: 1e-5
actor_lr: 1e-4
actor_weight_decay: 0
actor_lr_scheduler:
first_cycle_steps: 1000
warmup_steps: 10
min_lr: 1e-6
first_cycle_steps: ${train.n_train_itr}
warmup_steps: 0
min_lr: 1e-4
critic_lr: 1e-3
critic_weight_decay: 0
critic_lr_scheduler:
first_cycle_steps: 1000
warmup_steps: 10
first_cycle_steps: ${train.n_train_itr}
warmup_steps: 0
min_lr: 1e-3
save_model_freq: 100
val_freq: 10
@@ -75,7 +75,7 @@ train:
reward_scale_const: 1.0
gae_lambda: 0.95
batch_size: 10000
update_epochs: 8
update_epochs: 5
vf_coef: 0.5
target_kl: 1
@@ -87,7 +87,7 @@ model:
_target_: model.common.mlp_gmm.GMM_MLP
mlp_dims: [1024, 1024, 1024]
residual_style: True
fixed_std: 0.08
fixed_std: 0.1
learn_fixed_std: True
std_min: 0.01
std_max: 0.2
@@ -97,9 +97,9 @@ model:
action_dim: ${action_dim}
critic:
_target_: model.common.critic.CriticObs
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
mlp_dims: [256, 256, 256]
activation_type: Mish
residual_style: True
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
horizon_steps: ${horizon_steps}
device: ${device}
cfg/robomimic/finetune/transport/ft_ppo_gmm_transformer.yaml
+10 -10
View File
@@ -26,7 +26,7 @@ env:
name: ${env_name}
best_reward_threshold_for_success: 1
max_episode_steps: 800
save_video: false
save_video: False
wrappers:
robomimic_lowdim:
normalization_path: ${normalization_path}
@@ -49,21 +49,21 @@ wandb:
run: ${now:%H-%M-%S}_${name}
train:
n_train_itr: 1000
n_train_itr: 201
n_critic_warmup_itr: 2
n_steps: 400
gamma: 0.999
actor_lr: 1e-5
actor_lr: 1e-4
actor_weight_decay: 0
actor_lr_scheduler:
first_cycle_steps: 1000
warmup_steps: 10
min_lr: 1e-6
first_cycle_steps: ${train.n_train_itr}
warmup_steps: 0
min_lr: 1e-4
critic_lr: 1e-3
critic_weight_decay: 0
critic_lr_scheduler:
first_cycle_steps: 1000
warmup_steps: 10
first_cycle_steps: ${train.n_train_itr}
warmup_steps: 0
min_lr: 1e-3
save_model_freq: 100
val_freq: 10
@@ -75,7 +75,7 @@ train:
reward_scale_const: 1.0
gae_lambda: 0.95
batch_size: 10000
update_epochs: 8
update_epochs: 5
vf_coef: 0.5
target_kl: 1
@@ -98,9 +98,9 @@ model:
action_dim: ${action_dim}
critic:
_target_: model.common.critic.CriticObs
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
mlp_dims: [256, 256, 256]
activation_type: Mish
residual_style: True
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
horizon_steps: ${horizon_steps}
device: ${device}