dodox/dppo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

v0.7 (#26)

Browse Source 
* update from scratch configs

* update gym pretraining configs - use fewer epochs

* update robomimic pretraining configs - use fewer epochs

* allow trajectory plotting in eval agent

* add simple vit unet

* update avoid pretraining configs - use fewer epochs

* update furniture pretraining configs - use same amount of epochs as before

* add robomimic diffusion unet pretraining configs

* update robomimic finetuning configs - higher lr

* add vit unet checkpoint urls

* update pretraining and finetuning instructions as configs are updated
This commit is contained in:
Allen Z. Ren
2024-11-20 15:56:23 -05:00
committed by allenzren
parent d2929f65e1
commit 1d04211666
158 changed files with 3350 additions and 410 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cfg/gym/eval/halfcheetah-v2/eval_diffusion_mlp.yaml
+3 -3
View File
@@ -17,10 +17,10 @@ obs_dim: 17
action_dim: 6
denoising_steps: 20
cond_steps: 1
horizon_steps: 1
act_steps: 1
horizon_steps: 4
act_steps: 4
n_steps: 1000 # each episode can take maximum (max_episode_steps / act_steps, =250 right now) steps but may finish earlier in gym. We only count episodes finished within n_steps for evaluation.
n_steps: 250 # each episode can take maximum (max_episode_steps / act_steps, =250 right now) steps but may finish earlier in gym. We only count episodes finished within n_steps for evaluation.
render_num: 0
env:
cfg/gym/eval/hopper-v2/eval_diffusion_mlp.yaml
+1 -1
View File
@@ -20,7 +20,7 @@ cond_steps: 1
horizon_steps: 4
act_steps: 4
n_steps: 500 # each episode can take maximum (max_episode_steps / act_steps, =250 right now) steps but may finish earlier in gym. We only count episodes finished within n_steps for evaluation.
n_steps: 250 # each episode can take maximum (max_episode_steps / act_steps, =250 right now) steps but may finish earlier in gym. We only count episodes finished within n_steps for evaluation.
render_num: 0
env:
cfg/gym/eval/walker2d-v2/eval_diffusion_mlp.yaml
+61
View File
@@ -0,0 +1,61 @@
defaults:
- _self_
hydra:
run:
dir: ${logdir}
_target_: agent.eval.eval_diffusion_agent.EvalDiffusionAgent
name: ${env_name}_eval_diffusion_mlp_ta${horizon_steps}_td${denoising_steps}
logdir: ${oc.env:DPPO_LOG_DIR}/gym-eval/${name}/${now:%Y-%m-%d}_${now:%H-%M-%S}_${seed}
base_policy_path:
normalization_path: ${oc.env:DPPO_DATA_DIR}/gym/${env_name}/normalization.npz
seed: 42
device: cuda:0
env_name: walker2d-medium-v2
obs_dim: 17
action_dim: 6
denoising_steps: 20
cond_steps: 1
horizon_steps: 4
act_steps: 4
n_steps: 250 # each episode can take maximum (max_episode_steps / act_steps, =250 right now) steps but may finish earlier in gym. We only count episodes finished within n_steps for evaluation.
render_num: 0
env:
n_envs: 40
name: ${env_name}
max_episode_steps: 1000
reset_at_iteration: False
save_video: False
best_reward_threshold_for_success: 3 # success rate not relevant for gym tasks
wrappers:
mujoco_locomotion_lowdim:
normalization_path: ${normalization_path}
multi_step:
n_obs_steps: ${cond_steps}
n_action_steps: ${act_steps}
max_episode_steps: ${env.max_episode_steps}
reset_within_step: True
model:
_target_: model.diffusion.diffusion.DiffusionModel
predict_epsilon: True
denoised_clip_value: 1.0
#
network_path: ${base_policy_path}
network:
_target_: model.diffusion.mlp_diffusion.DiffusionMLP
time_dim: 16
mlp_dims: [512, 512, 512]
activation_type: ReLU
residual_style: True
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
horizon_steps: ${horizon_steps}
action_dim: ${action_dim}
horizon_steps: ${horizon_steps}
obs_dim: ${obs_dim}
action_dim: ${action_dim}
denoising_steps: ${denoising_steps}
device: ${device}
cfg/gym/pretrain/halfcheetah-medium-v2/pre_diffusion_mlp.yaml
+2 -2
View File
@@ -24,12 +24,12 @@ wandb:
run: ${now:%H-%M-%S}_${name}
train:
n_epochs: 3000
n_epochs: 200
batch_size: 128
learning_rate: 1e-3
weight_decay: 1e-6
lr_scheduler:
first_cycle_steps: 3000
first_cycle_steps: 200
warmup_steps: 1
min_lr: 1e-4
save_model_freq: 100
cfg/gym/pretrain/halfcheetah-medium-v2/pre_gaussian_mlp.yaml
+2 -3
View File
@@ -23,15 +23,14 @@ wandb:
run: ${now:%H-%M-%S}_${name}
train:
n_epochs: 500
n_epochs: 200
batch_size: 128
learning_rate: 1e-4
weight_decay: 1e-6
lr_scheduler:
first_cycle_steps: 1000
first_cycle_steps: 200
warmup_steps: 1
min_lr: 1e-4
save_model_freq: 100
model:
cfg/gym/pretrain/hopper-medium-v2/pre_diffusion_mlp.yaml
+2 -2
View File
@@ -24,12 +24,12 @@ wandb:
run: ${now:%H-%M-%S}_${name}
train:
n_epochs: 3000
n_epochs: 200
batch_size: 128
learning_rate: 1e-3
weight_decay: 1e-6
lr_scheduler:
first_cycle_steps: 3000
first_cycle_steps: 200
warmup_steps: 1
min_lr: 1e-4
save_model_freq: 100
cfg/gym/pretrain/hopper-medium-v2/pre_gaussian_mlp.yaml
+2 -2
View File
@@ -23,12 +23,12 @@ wandb:
run: ${now:%H-%M-%S}_${name}
train:
n_epochs: 500
n_epochs: 200
batch_size: 128
learning_rate: 1e-4
weight_decay: 1e-6
lr_scheduler:
first_cycle_steps: 1000
first_cycle_steps: 200
warmup_steps: 1
min_lr: 1e-4
save_model_freq: 100
cfg/gym/pretrain/kitchen-complete-v0/pre_diffusion_mlp.yaml
+2 -2
View File
@@ -24,12 +24,12 @@ wandb:
run: ${now:%H-%M-%S}_${name}
train:
n_epochs: 8000
n_epochs: 3000
batch_size: 128
learning_rate: 1e-3
weight_decay: 1e-6
lr_scheduler:
first_cycle_steps: 8000
first_cycle_steps: 3000
warmup_steps: 1
min_lr: 1e-4
save_model_freq: 500
cfg/gym/pretrain/kitchen-complete-v0/pre_gaussian_mlp.yaml
+2 -2
View File
@@ -23,12 +23,12 @@ wandb:
run: ${now:%H-%M-%S}_${name}
train:
n_epochs: 5000
n_epochs: 3000
batch_size: 256
learning_rate: 1e-4
weight_decay: 0
lr_scheduler:
first_cycle_steps: 5000
first_cycle_steps: 3000
warmup_steps: 100
min_lr: 1e-4
save_model_freq: 500
cfg/gym/pretrain/kitchen-mixed-v0/pre_diffusion_mlp.yaml
+2 -2
View File
@@ -24,12 +24,12 @@ wandb:
run: ${now:%H-%M-%S}_${name}
train:
n_epochs: 8000
n_epochs: 3000
batch_size: 256
learning_rate: 1e-3
weight_decay: 1e-6
lr_scheduler:
first_cycle_steps: 8000
first_cycle_steps: 3000
warmup_steps: 1
min_lr: 1e-4
save_model_freq: 500
cfg/gym/pretrain/kitchen-mixed-v0/pre_gaussian_mlp.yaml
+2 -2
View File
@@ -23,12 +23,12 @@ wandb:
run: ${now:%H-%M-%S}_${name}
train:
n_epochs: 5000
n_epochs: 3000
batch_size: 128
learning_rate: 1e-3
weight_decay: 1e-6
lr_scheduler:
first_cycle_steps: 5000
first_cycle_steps: 3000
warmup_steps: 1
min_lr: 1e-4
save_model_freq: 500
cfg/gym/pretrain/kitchen-partial-v0/pre_diffusion_mlp.yaml
+2 -2
View File
@@ -24,12 +24,12 @@ wandb:
run: ${now:%H-%M-%S}_${name}
train:
n_epochs: 8000
n_epochs: 3000
batch_size: 128
learning_rate: 1e-3
weight_decay: 1e-5
lr_scheduler:
first_cycle_steps: 8000
first_cycle_steps: 3000
warmup_steps: 1
min_lr: 1e-4
save_model_freq: 500
cfg/gym/pretrain/kitchen-partial-v0/pre_gaussian_mlp.yaml
+2 -2
View File
@@ -23,12 +23,12 @@ wandb:
run: ${now:%H-%M-%S}_${name}
train:
n_epochs: 5000
n_epochs: 3000
batch_size: 128
learning_rate: 1e-3
weight_decay: 1e-6
lr_scheduler:
first_cycle_steps: 5000
first_cycle_steps: 3000
warmup_steps: 1
min_lr: 1e-4
save_model_freq: 500
cfg/gym/pretrain/walker2d-medium-v2/pre_diffusion_mlp.yaml
+2 -2
View File
@@ -24,12 +24,12 @@ wandb:
run: ${now:%H-%M-%S}_${name}
train:
n_epochs: 3000
n_epochs: 200
batch_size: 128
learning_rate: 1e-3
weight_decay: 1e-6
lr_scheduler:
first_cycle_steps: 3000
first_cycle_steps: 200
warmup_steps: 1
min_lr: 1e-4
save_model_freq: 100
cfg/gym/pretrain/walker2d-medium-v2/pre_gaussian_mlp.yaml
+2 -2
View File
@@ -23,12 +23,12 @@ wandb:
run: ${now:%H-%M-%S}_${name}
train:
n_epochs: 3000
n_epochs: 200
batch_size: 128
learning_rate: 1e-4
weight_decay: 1e-6
lr_scheduler:
first_cycle_steps: 3000
first_cycle_steps: 200
warmup_steps: 1
min_lr: 1e-4
save_model_freq: 100
cfg/gym/scratch/halfcheetah-v2/ppo_diffusion_mlp.yaml
+6 -6
View File
@@ -1,7 +1,7 @@
defaults:
- _self_
hydra:
run:
run:
dir: ${logdir}
_target_: agent.finetune.train_ppo_diffusion_agent.TrainPPODiffusionAgent
@@ -42,7 +42,7 @@ wandb:
run: ${now:%H-%M-%S}_${name}
train:
n_train_itr: 1000
n_train_itr: 501
n_critic_warmup_itr: 0
n_steps: 1000
gamma: 0.99
@@ -55,7 +55,7 @@ train:
critic_lr: 1e-3
critic_weight_decay: 0
critic_lr_scheduler:
first_cycle_steps: 10000
first_cycle_steps: 1000
warmup_steps: 10
min_lr: 1e-3
save_model_freq: 100
@@ -67,7 +67,7 @@ train:
reward_scale_running: True
reward_scale_const: 1.0
gae_lambda: 0.95
batch_size: 10000
batch_size: 5000
update_epochs: 10
vf_coef: 0.5
target_kl: 1
@@ -75,7 +75,7 @@ train:
model:
_target_: model.diffusion.diffusion_ppo.PPODiffusion
# HP to tune
gamma_denoising: 0.99
gamma_denoising: 1
clip_ploss_coef: 0.1
clip_ploss_coef_base: 0.1
clip_ploss_coef_rate: 3
@@ -94,10 +94,10 @@ model:
residual_style: True
critic:
_target_: model.common.critic.CriticObs
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
mlp_dims: [256, 256, 256]
activation_type: Mish
residual_style: True
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
ft_denoising_steps: ${ft_denoising_steps}
horizon_steps: ${horizon_steps}
obs_dim: ${obs_dim}
cfg/gym/scratch/halfcheetah-v2/ppo_gaussian_mlp.yaml
+2 -2
View File
@@ -40,7 +40,7 @@ wandb:
run: ${now:%H-%M-%S}_${name}
train:
n_train_itr: 1000
n_train_itr: 501
n_critic_warmup_itr: 0
n_steps: 1000
gamma: 0.99
@@ -65,7 +65,7 @@ train:
reward_scale_running: True
reward_scale_const: 1.0
gae_lambda: 0.95
batch_size: 1000
batch_size: 500
update_epochs: 10
vf_coef: 0.5
target_kl: 1
cfg/gym/scratch/hopper-v2/ppo_diffusion_mlp.yaml
+3 -3
View File
@@ -42,7 +42,7 @@ wandb:
run: ${now:%H-%M-%S}_${name}
train:
n_train_itr: 1000
n_train_itr: 301
n_critic_warmup_itr: 0
n_steps: 1000
gamma: 0.99
@@ -67,7 +67,7 @@ train:
reward_scale_running: True
reward_scale_const: 1.0
gae_lambda: 0.95
batch_size: 10000
batch_size: 5000
update_epochs: 10
vf_coef: 0.5
target_kl: 1
@@ -75,7 +75,7 @@ train:
model:
_target_: model.diffusion.diffusion_ppo.PPODiffusion
# HP to tune
gamma_denoising: 0.99
gamma_denoising: 1
clip_ploss_coef: 0.1
clip_ploss_coef_base: 0.1
clip_ploss_coef_rate: 3
cfg/gym/scratch/hopper-v2/ppo_gaussian_mlp.yaml
+2 -2
View File
@@ -40,7 +40,7 @@ wandb:
run: ${now:%H-%M-%S}_${name}
train:
n_train_itr: 1000
n_train_itr: 301
n_critic_warmup_itr: 0
n_steps: 1000
gamma: 0.99
@@ -65,7 +65,7 @@ train:
reward_scale_running: True
reward_scale_const: 1.0
gae_lambda: 0.95
batch_size: 1000
batch_size: 500
update_epochs: 10
vf_coef: 0.5
target_kl: 1
cfg/gym/scratch/walker2d-v2/ppo_diffusion_mlp.yaml
+5 -5
View File
@@ -42,7 +42,7 @@ wandb:
run: ${now:%H-%M-%S}_${name}
train:
n_train_itr: 1000
n_train_itr: 501
n_critic_warmup_itr: 0
n_steps: 1000
gamma: 0.99
@@ -55,7 +55,7 @@ train:
critic_lr: 1e-3
critic_weight_decay: 0
critic_lr_scheduler:
first_cycle_steps: 10000
first_cycle_steps: 1000
warmup_steps: 10
min_lr: 1e-3
save_model_freq: 100
@@ -67,7 +67,7 @@ train:
reward_scale_running: True
reward_scale_const: 1.0
gae_lambda: 0.95
batch_size: 10000
batch_size: 5000
update_epochs: 10
vf_coef: 0.5
target_kl: 1
@@ -75,7 +75,7 @@ train:
model:
_target_: model.diffusion.diffusion_ppo.PPODiffusion
# HP to tune
gamma_denoising: 0.99
gamma_denoising: 1
clip_ploss_coef: 0.1
clip_ploss_coef_base: 0.1
clip_ploss_coef_rate: 3
@@ -94,10 +94,10 @@ model:
residual_style: True
critic:
_target_: model.common.critic.CriticObs
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
mlp_dims: [256, 256, 256]
activation_type: Mish
residual_style: True
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
ft_denoising_steps: ${ft_denoising_steps}
horizon_steps: ${horizon_steps}
obs_dim: ${obs_dim}
cfg/gym/scratch/walker2d-v2/ppo_gaussian_mlp.yaml
+2 -2
View File
@@ -40,7 +40,7 @@ wandb:
run: ${now:%H-%M-%S}_${name}
train:
n_train_itr: 1000
n_train_itr: 301
n_critic_warmup_itr: 0
n_steps: 1000
gamma: 0.99
@@ -65,7 +65,7 @@ train:
reward_scale_running: True
reward_scale_const: 1.0
gae_lambda: 0.95
batch_size: 1000
batch_size: 500
update_epochs: 10
vf_coef: 0.5
target_kl: 1