v0.5 to main (#10)

* v0.5 (#9)

* update idql configs

* update awr configs

* update dipo configs

* update qsm configs

* update dqm configs

* update project version to 0.5.0

cfg/robomimic/finetune/transport/ft_awr_diffusion_mlp.yaml

@@ -16,7 +16,6 @@ device: cuda:0
 env_name: transport
 obs_dim: 59
 action_dim: 14
-transition_dim: ${action_dim}
 denoising_steps: 20
 cond_steps: 1
 horizon_steps: 8
@@ -76,7 +75,7 @@ train:
   max_adv_weight: 100
   beta: 10
   buffer_size: 3000
-  batch_size: 256
+  batch_size: 1000
   replay_ratio: 64
   critic_update_ratio: 4
 
@@ -94,13 +93,13 @@ model:
     residual_style: True
     cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
     horizon_steps: ${horizon_steps}
-    transition_dim: ${transition_dim}
+    action_dim: ${action_dim}
   critic:
     _target_: model.common.critic.CriticObs
-    cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
     mlp_dims: [256, 256, 256]
     activation_type: Mish
     residual_style: True
+    cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
   horizon_steps: ${horizon_steps}
   obs_dim: ${obs_dim}
   action_dim: ${action_dim}

cfg/robomimic/finetune/transport/ft_dipo_diffusion_mlp.yaml

@@ -16,7 +16,6 @@ device: cuda:0
 env_name: transport
 obs_dim: 59
 action_dim: 14
-transition_dim: ${action_dim}
 denoising_steps: 20
 cond_steps: 1
 horizon_steps: 8
@@ -73,11 +72,12 @@ train:
     num: 0
   # DIPO specific
   scale_reward_factor: 1
-  eta: 0.0001
+  target_ema_rate: 0.005
+  buffer_size: 1000000
+  action_lr: 0.0001
   action_gradient_steps: 10
-  buffer_size: 400000
-  batch_size: 5000
-  replay_ratio: 64
+  replay_ratio: 16
+  batch_size: 1000
 
 model:
   _target_: model.diffusion.diffusion_dipo.DIPODiffusion
@@ -93,7 +93,7 @@ model:
     residual_style: True
     cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
     horizon_steps: ${horizon_steps}
-    transition_dim: ${transition_dim}
+    action_dim: ${action_dim}
   critic:
     _target_: model.common.critic.CriticObsAct
     action_dim: ${action_dim}

cfg/robomimic/finetune/transport/ft_dql_diffusion_mlp.yaml

@@ -16,7 +16,6 @@ device: cuda:0
 env_name: transport
 obs_dim: 59
 action_dim: 14
-transition_dim: ${action_dim}
 denoising_steps: 20
 cond_steps: 1
 horizon_steps: 8
@@ -73,10 +72,11 @@ train:
     num: 0
   # DQL specific
   scale_reward_factor: 1
+  target_ema_rate: 0.005
+  buffer_size: 1000000
   eta: 1.0
-  buffer_size: 400000
-  batch_size: 5000
-  replay_ratio: 64
+  replay_ratio: 16
+  batch_size: 1000
 
 model:
   _target_: model.diffusion.diffusion_dql.DQLDiffusion
@@ -92,7 +92,7 @@ model:
     residual_style: True
     cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
     horizon_steps: ${horizon_steps}
-    transition_dim: ${transition_dim}
+    action_dim: ${action_dim}
   critic:
     _target_: model.common.critic.CriticObsAct
     action_dim: ${action_dim}

cfg/robomimic/finetune/transport/ft_idql_diffusion_mlp.yaml

@@ -16,7 +16,6 @@ device: cuda:0
 env_name: transport
 obs_dim: 59
 action_dim: 14
-transition_dim: ${action_dim}
 denoising_steps: 20
 cond_steps: 1
 horizon_steps: 8
@@ -77,9 +76,9 @@ train:
   eval_sample_num: 10 # how many samples to score during eval
   critic_tau: 0.001  # rate of target q network update
   use_expectile_exploration: True
-  buffer_size: 3000
-  batch_size: 512
-  replay_ratio: 16
+  buffer_size: 5000 # * n_envs
+  replay_ratio: 128
+  batch_size: 1000
 
 model:
   _target_: model.diffusion.diffusion_idql.IDQLDiffusion
@@ -95,7 +94,7 @@ model:
     residual_style: True
     cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
     horizon_steps: ${horizon_steps}
-    transition_dim: ${transition_dim}
+    action_dim: ${action_dim}
   critic_q:
     _target_: model.common.critic.CriticObsAct
     action_dim: ${action_dim}

cfg/robomimic/finetune/transport/ft_ppo_diffusion_mlp.yaml

@@ -16,7 +16,6 @@ device: cuda:0
 env_name: transport
 obs_dim: 59
 action_dim: 14
-transition_dim: ${action_dim}
 denoising_steps: 20
 ft_denoising_steps: 10
 cond_steps: 1
@@ -100,7 +99,7 @@ model:
     residual_style: True
     cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
     horizon_steps: ${horizon_steps}
-    transition_dim: ${transition_dim}
+    action_dim: ${action_dim}
   critic:
     _target_: model.common.critic.CriticObs
     cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}

cfg/robomimic/finetune/transport/ft_ppo_diffusion_mlp_img.yaml

@@ -16,7 +16,6 @@ device: cuda:0
 env_name: transport
 obs_dim: 18
 action_dim: 14
-transition_dim: ${action_dim}
 denoising_steps: 100
 ft_denoising_steps: 5
 cond_steps: 1
@@ -145,7 +144,7 @@ model:
     img_cond_steps: ${img_cond_steps}
     cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
     horizon_steps: ${horizon_steps}
-    transition_dim: ${transition_dim}
+    action_dim: ${action_dim}
   critic:
     _target_: model.common.critic.ViTCritic
     spatial_emb: 128

cfg/robomimic/finetune/transport/ft_ppo_diffusion_unet.yaml

@@ -16,7 +16,6 @@ device: cuda:0
 env_name: transport
 obs_dim: 59
 action_dim: 14
-transition_dim: ${action_dim}
 denoising_steps: 20
 ft_denoising_steps: 10
 cond_steps: 1
@@ -103,7 +102,7 @@ model:
     smaller_encoder: False
     cond_predict_scale: True
     cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
-    transition_dim: ${transition_dim}
+    action_dim: ${action_dim}
   critic:
     _target_: model.common.critic.CriticObs
     cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}

cfg/robomimic/finetune/transport/ft_ppo_gaussian_mlp.yaml

@@ -16,7 +16,6 @@ device: cuda:0
 env_name: transport
 obs_dim: 59
 action_dim: 14
-transition_dim: ${action_dim}
 cond_steps: 1
 horizon_steps: 8
 act_steps: 8
@@ -94,7 +93,7 @@ model:
     std_max: 0.2
     cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
     horizon_steps: ${horizon_steps}
-    transition_dim: ${transition_dim}
+    action_dim: ${action_dim}
   critic:
     _target_: model.common.critic.CriticObs
     cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}

cfg/robomimic/finetune/transport/ft_ppo_gaussian_mlp_img.yaml

@@ -16,7 +16,6 @@ device: cuda:0
 env_name: transport
 obs_dim: 18
 action_dim: 14
-transition_dim: ${action_dim}
 cond_steps: 1
 img_cond_steps: 1
 horizon_steps: 8
@@ -127,7 +126,7 @@ model:
     img_cond_steps: ${img_cond_steps}
     cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
     horizon_steps: ${horizon_steps}
-    transition_dim: ${transition_dim}
+    action_dim: ${action_dim}
   critic:
     _target_: model.common.critic.ViTCritic
     spatial_emb: 128

cfg/robomimic/finetune/transport/ft_ppo_gaussian_transformer.yaml

@@ -16,7 +16,6 @@ device: cuda:0
 env_name: transport
 obs_dim: 59
 action_dim: 14
-transition_dim: ${action_dim}
 cond_steps: 1
 horizon_steps: 8
 act_steps: 8
@@ -95,7 +94,7 @@ model:
     std_max: 0.2
     cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
     horizon_steps: ${horizon_steps}
-    transition_dim: ${transition_dim}
+    action_dim: ${action_dim}
   critic:
     _target_: model.common.critic.CriticObs
     cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}

cfg/robomimic/finetune/transport/ft_ppo_gmm_mlp.yaml

@@ -16,7 +16,6 @@ device: cuda:0
 env_name: transport
 obs_dim: 59
 action_dim: 14
-transition_dim: ${action_dim}
 cond_steps: 1
 horizon_steps: 8
 act_steps: 8
@@ -95,7 +94,7 @@ model:
     num_modes: ${num_modes}
     cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
     horizon_steps: ${horizon_steps}
-    transition_dim: ${transition_dim}
+    action_dim: ${action_dim}
   critic:
     _target_: model.common.critic.CriticObs
     cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}

cfg/robomimic/finetune/transport/ft_ppo_gmm_transformer.yaml

@@ -16,7 +16,6 @@ device: cuda:0
 env_name: transport
 obs_dim: 59
 action_dim: 14
-transition_dim: ${action_dim}
 cond_steps: 1
 horizon_steps: 8
 act_steps: 8
@@ -96,7 +95,7 @@ model:
     num_modes: ${num_modes}
     cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
     horizon_steps: ${horizon_steps}
-    transition_dim: ${transition_dim}
+    action_dim: ${action_dim}
   critic:
     _target_: model.common.critic.CriticObs
     cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}

cfg/robomimic/finetune/transport/ft_qsm_diffusion_mlp.yaml

@@ -16,7 +16,6 @@ device: cuda:0
 env_name: transport
 obs_dim: 59
 action_dim: 14
-transition_dim: ${action_dim}
 denoising_steps: 20
 cond_steps: 1
 horizon_steps: 8
@@ -73,11 +72,11 @@ train:
     num: 0
   # QSM specific
   scale_reward_factor: 1
-  q_grad_coeff: 50
+  q_grad_coeff: 10
   critic_tau: 0.005  # rate of target q network update
-  buffer_size: 3000
-  batch_size: 256
-  replay_ratio: 32
+  buffer_size: 5000 # * n_envs
+  replay_ratio: 16
+  batch_size: 1000
 
 model:
   _target_: model.diffusion.diffusion_qsm.QSMDiffusion
@@ -93,7 +92,7 @@ model:
     residual_style: True
     cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
     horizon_steps: ${horizon_steps}
-    transition_dim: ${transition_dim}
+    action_dim: ${action_dim}
   critic:
     _target_: model.common.critic.CriticObsAct
     action_dim: ${action_dim}

cfg/robomimic/finetune/transport/ft_rwr_diffusion_mlp.yaml

@@ -16,7 +16,6 @@ device: cuda:0
 env_name: transport
 obs_dim: 59
 action_dim: 14
-transition_dim: ${action_dim}
 denoising_steps: 20
 cond_steps: 1
 horizon_steps: 8
@@ -85,7 +84,7 @@ model:
     residual_style: True
     cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
     horizon_steps: ${horizon_steps}
-    transition_dim: ${transition_dim}
+    action_dim: ${action_dim}
   horizon_steps: ${horizon_steps}
   obs_dim: ${obs_dim}
   action_dim: ${action_dim}