dodox/dppo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

release

Browse Source
This commit is contained in:
allenzren
2024-09-03 21:03:27 -04:00
commit 8293b0936b
282 changed files with 34664 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cfg/d3il/finetune/avoid_m1/ft_ppo_diffusion_mlp.yaml
+119
View File
@@ -0,0 +1,119 @@
defaults:
- _self_
hydra:
run:
dir: ${logdir}
_target_: agent.finetune.train_ppo_diffusion_agent.TrainPPODiffusionAgent
name: ${env_name}_m1_ft_diffusion_mlp_ta${horizon_steps}_td${denoising_steps}_tdf${ft_denoising_steps}
logdir: ${oc.env:DPPO_LOG_DIR}/d3il-finetune/${name}/${now:%Y-%m-%d}_${now:%H-%M-%S}_${seed}
base_policy_path: ${oc.env:DPPO_LOG_DIR}/d3il-pretrain/m1/avoid_d56_r12_pre_diffusion_mlp_ta4_td20/2024-07-06_22-50-07/checkpoint/state_10000.pt
normalization_path: ${oc.env:DPPO_DATA_DIR}/d3il/avoid_m1/normalization.npz
seed: 42
device: cuda:0
env_name: avoiding-m5
mode: d56_r12 # M1, desired modes 5 and 6, required modes 1 and 2
obs_dim: 4
action_dim: 2
transition_dim: ${action_dim}
denoising_steps: 20
ft_denoising_steps: 10
cond_steps: 1
horizon_steps: 4
act_steps: 4
env:
n_envs: 50
name: ${env_name}
max_episode_steps: 100
reset_at_iteration: True
save_video: False
best_reward_threshold_for_success: 2
save_full_observations: True
wrappers:
d3il_lowdim:
normalization_path: ${normalization_path}
multi_step:
n_obs_steps: ${cond_steps}
n_action_steps: ${act_steps}
max_episode_steps: ${env.max_episode_steps}
pass_full_observations: ${env.save_full_observations}
reset_within_step: False
wandb:
entity: ${oc.env:DPPO_WANDB_ENTITY}
project: d3il-${env_name}-m1-finetune
run: ${now:%H-%M-%S}_${name}
train:
n_train_itr: 51
n_critic_warmup_itr: 1
n_steps: ${eval:'round(${env.max_episode_steps} / ${act_steps})'}
gamma: 0.99
actor_lr: 1e-5
actor_weight_decay: 0
actor_lr_scheduler:
first_cycle_steps: 100
warmup_steps: 10
min_lr: 1e-5
critic_lr: 1e-3
critic_weight_decay: 0
critic_lr_scheduler:
first_cycle_steps: 100
warmup_steps: 10
min_lr: 1e-3
save_model_freq: 100
val_freq: 100 # no eval, always train mode
force_train: True
render:
freq: 1
num: 10
plotter:
_target_: env.plot_traj.TrajPlotter
env_type: avoid
normalization_path: ${normalization_path}
# PPO specific
reward_scale_running: True
reward_scale_const: 1.0
gae_lambda: 0.95
batch_size: ${eval:'round(${train.n_steps} * ${env.n_envs} / 2)'}
update_epochs: 10
vf_coef: 0.5
target_kl: 1
model:
_target_: model.diffusion.diffusion_ppo.PPODiffusion
# HP to tune
gamma_denoising: 0.95
clip_ploss_coef: 0.1
clip_ploss_coef_base: 0.1
clip_ploss_coef_rate: 1
randn_clip_value: 3
min_sampling_denoising_std: 0.1
min_logprob_denoising_std: 0.1
#
network_path: ${base_policy_path}
actor:
_target_: model.diffusion.mlp_diffusion.DiffusionMLP
time_dim: 16
mlp_dims: [512, 512, 512]
activation_type: ReLU
residual_style: True
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
horizon_steps: ${horizon_steps}
transition_dim: ${transition_dim}
critic:
_target_: model.common.critic.CriticObs
obs_dim: ${obs_dim}
mlp_dims: [256, 256, 256]
activation_type: Mish
residual_style: True
ft_denoising_steps: ${ft_denoising_steps}
transition_dim: ${transition_dim}
horizon_steps: ${horizon_steps}
obs_dim: ${obs_dim}
action_dim: ${action_dim}
cond_steps: ${cond_steps}
denoising_steps: ${denoising_steps}
device: ${device}
cfg/d3il/finetune/avoid_m1/ft_ppo_gaussian_mlp.yaml
+105
View File
@@ -0,0 +1,105 @@
defaults:
- _self_
hydra:
run:
dir: ${logdir}
_target_: agent.finetune.train_ppo_gaussian_agent.TrainPPOGaussianAgent
name: ${env_name}_m1_gaussian_mlp_ta${horizon_steps}
logdir: ${oc.env:DPPO_LOG_DIR}/d3il-finetune/${name}/${now:%Y-%m-%d}_${now:%H-%M-%S}_${seed}
base_policy_path: ${oc.env:DPPO_LOG_DIR}/d3il-pretrain/m1/avoid_d56_r12_pre_gaussian_mlp_ta4/2024-07-07_01-35-48/checkpoint/state_10000.pt
normalization_path: ${oc.env:DPPO_DATA_DIR}/d3il/avoid_m1/normalization.npz
seed: 42
device: cuda:0
env_name: avoiding-m5
mode: d56_r12 # M1, desired modes 5 and 6, required modes 1 and 2
obs_dim: 4
action_dim: 2
transition_dim: ${action_dim}
cond_steps: 1
horizon_steps: 4
act_steps: 4
env:
n_envs: 50
name: ${env_name}
max_episode_steps: 100
reset_at_iteration: True
save_video: False
best_reward_threshold_for_success: 2
save_full_observations: True
wrappers:
d3il_lowdim:
normalization_path: ${normalization_path}
multi_step:
n_obs_steps: ${cond_steps}
n_action_steps: ${act_steps}
max_episode_steps: ${env.max_episode_steps}
pass_full_observations: ${env.save_full_observations}
reset_within_step: False
wandb:
entity: ${oc.env:DPPO_WANDB_ENTITY}
project: d3il-${env_name}-m1-finetune
run: ${now:%H-%M-%S}_${name}
train:
n_train_itr: 51
n_critic_warmup_itr: 1
n_steps: ${eval:'round(${env.max_episode_steps} / ${act_steps})'}
gamma: 0.99
actor_lr: 1e-5
actor_weight_decay: 0
actor_lr_scheduler:
first_cycle_steps: 100
warmup_steps: 10
min_lr: 1e-5
critic_lr: 1e-3
critic_weight_decay: 0
critic_lr_scheduler:
first_cycle_steps: 100
warmup_steps: 10
min_lr: 1e-3
save_model_freq: 100
val_freq: 100 # no eval, always train mode
force_train: True
render:
freq: 1
num: 10
plotter:
_target_: env.plot_traj.TrajPlotter
env_type: avoid
normalization_path: ${normalization_path}
# PPO specific
reward_scale_running: True
reward_scale_const: 1.0
gae_lambda: 0.95
batch_size: ${eval:'round(${train.n_steps} * ${env.n_envs} / 2)'}
update_epochs: 10
vf_coef: 0.5
target_kl: 1
model:
_target_: model.rl.gaussian_ppo.PPO_Gaussian
clip_ploss_coef: 0.1
randn_clip_value: 3
network_path: ${base_policy_path}
actor:
_target_: model.common.mlp_gaussian.Gaussian_MLP
mlp_dims: [256, 256, 256] # smaller MLP for less overfitting
activation_type: ReLU
residual_style: True
fixed_std: 0.1
learn_fixed_std: False
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
horizon_steps: ${horizon_steps}
transition_dim: ${transition_dim}
critic:
_target_: model.common.critic.CriticObs
mlp_dims: [256, 256, 256]
residual_style: True
obs_dim: ${obs_dim}
horizon_steps: ${horizon_steps}
cond_steps: ${cond_steps}
device: ${device}
cfg/d3il/finetune/avoid_m1/ft_ppo_gmm_mlp.yaml
+106
View File
@@ -0,0 +1,106 @@
defaults:
- _self_
hydra:
run:
dir: ${logdir}
_target_: agent.finetune.train_ppo_gaussian_agent.TrainPPOGaussianAgent
name: ${env_name}_m1_gmm_mlp_ta${horizon_steps}
logdir: ${oc.env:DPPO_LOG_DIR}/d3il-finetune/${name}/${now:%Y-%m-%d}_${now:%H-%M-%S}_${seed}
base_policy_path: ${oc.env:DPPO_LOG_DIR}/d3il-pretrain/m1/avoid_d56_r12_pre_gmm_mlp_ta4/2024-07-10_14-30-00/checkpoint/state_10000.pt
normalization_path: ${oc.env:DPPO_DATA_DIR}/d3il/avoid_m1/normalization.npz
seed: 42
device: cuda:0
env_name: avoiding-m5
mode: d56_r12 # M1, desired modes 5 and 6, required modes 1 and 2
obs_dim: 4
action_dim: 2
transition_dim: ${action_dim}
cond_steps: 1
horizon_steps: 4
act_steps: 4
num_modes: 5
env:
n_envs: 50
name: ${env_name}
max_episode_steps: 100
reset_at_iteration: True
save_video: False
best_reward_threshold_for_success: 2
save_full_observations: True
wrappers:
d3il_lowdim:
normalization_path: ${normalization_path}
multi_step:
n_obs_steps: ${cond_steps}
n_action_steps: ${act_steps}
max_episode_steps: ${env.max_episode_steps}
pass_full_observations: ${env.save_full_observations}
reset_within_step: False
wandb:
entity: ${oc.env:DPPO_WANDB_ENTITY}
project: d3il-${env_name}-m1-finetune
run: ${now:%H-%M-%S}_${name}
train:
n_train_itr: 51
n_critic_warmup_itr: 1
n_steps: ${eval:'round(${env.max_episode_steps} / ${act_steps})'}
gamma: 0.99
actor_lr: 1e-5
actor_weight_decay: 0
actor_lr_scheduler:
first_cycle_steps: 100
warmup_steps: 10
min_lr: 1e-5
critic_lr: 1e-3
critic_weight_decay: 0
critic_lr_scheduler:
first_cycle_steps: 100
warmup_steps: 10
min_lr: 1e-3
save_model_freq: 100
val_freq: 100 # no eval, always train mode
force_train: True
render:
freq: 1
num: 10
plotter:
_target_: env.plot_traj.TrajPlotter
env_type: avoid
normalization_path: ${normalization_path}
# PPO specific
reward_scale_running: True
reward_scale_const: 1.0
gae_lambda: 0.95
batch_size: ${eval:'round(${train.n_steps} * ${env.n_envs} / 2)'}
update_epochs: 10
vf_coef: 0.5
target_kl: 1
model:
_target_: model.rl.gmm_ppo.PPO_GMM
clip_ploss_coef: 0.1
network_path: ${base_policy_path}
actor:
_target_: model.common.mlp_gmm.GMM_MLP
mlp_dims: [256, 256] # smaller MLP for less overfitting
activation_type: ReLU
residual_style: False
fixed_std: 0.1
learn_fixed_std: False
num_modes: ${num_modes}
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
horizon_steps: ${horizon_steps}
transition_dim: ${transition_dim}
critic:
_target_: model.common.critic.CriticObs
mlp_dims: [256, 256, 256]
residual_style: True
obs_dim: ${obs_dim}
horizon_steps: ${horizon_steps}
cond_steps: ${cond_steps}
device: ${device}
cfg/d3il/finetune/avoid_m2/ft_ppo_diffusion_mlp.yaml
+119
View File
@@ -0,0 +1,119 @@
defaults:
- _self_
hydra:
run:
dir: ${logdir}
_target_: agent.finetune.train_ppo_diffusion_agent.TrainPPODiffusionAgent
name: ${env_name}_m2_ft_diffusion_mlp_ta${horizon_steps}_td${denoising_steps}_tdf${ft_denoising_steps}
logdir: ${oc.env:DPPO_LOG_DIR}/d3il-finetune/${name}/${now:%Y-%m-%d}_${now:%H-%M-%S}_${seed}
base_policy_path: ${oc.env:DPPO_LOG_DIR}/d3il-pretrain/m2/avoid_d57_r12_pre_diffusion_mlp_ta4_td20/2024-07-07_13-12-09/checkpoint/state_15000.pt
normalization_path: ${oc.env:DPPO_DATA_DIR}/d3il/avoid_m2/normalization.npz
seed: 42
device: cuda:0
env_name: avoiding-m5
mode: d57_r12 # M2, desired modes 5 and 7, required modes 1 and 2
obs_dim: 4
action_dim: 2
transition_dim: ${action_dim}
denoising_steps: 20
ft_denoising_steps: 10
cond_steps: 1
horizon_steps: 4
act_steps: 4
env:
n_envs: 50
name: ${env_name}
max_episode_steps: 100
reset_at_iteration: True
save_video: False
best_reward_threshold_for_success: 2
save_full_observations: True
wrappers:
d3il_lowdim:
normalization_path: ${normalization_path}
multi_step:
n_obs_steps: ${cond_steps}
n_action_steps: ${act_steps}
max_episode_steps: ${env.max_episode_steps}
pass_full_observations: ${env.save_full_observations}
reset_within_step: False
wandb:
entity: ${oc.env:DPPO_WANDB_ENTITY}
project: d3il-${env_name}-m2-finetune
run: ${now:%H-%M-%S}_${name}
train:
n_train_itr: 51
n_critic_warmup_itr: 1
n_steps: ${eval:'round(${env.max_episode_steps} / ${act_steps})'}
gamma: 0.99
actor_lr: 1e-5
actor_weight_decay: 0
actor_lr_scheduler:
first_cycle_steps: 100
warmup_steps: 10
min_lr: 1e-5
critic_lr: 1e-3
critic_weight_decay: 0
critic_lr_scheduler:
first_cycle_steps: 100
warmup_steps: 10
min_lr: 1e-3
save_model_freq: 100
val_freq: 100 # no eval, always train mode
force_train: True
render:
freq: 1
num: 10
plotter:
_target_: env.plot_traj.TrajPlotter
env_type: avoid
normalization_path: ${normalization_path}
# PPO specific
reward_scale_running: True
reward_scale_const: 1.0
gae_lambda: 0.95
batch_size: ${eval:'round(${train.n_steps} * ${env.n_envs} / 2)'}
update_epochs: 10
vf_coef: 0.5
target_kl: 1
model:
_target_: model.diffusion.diffusion_ppo.PPODiffusion
# HP to tune
gamma_denoising: 0.95
clip_ploss_coef: 0.1
clip_ploss_coef_base: 0.1
clip_ploss_coef_rate: 1
randn_clip_value: 3
min_sampling_denoising_std: 0.1
min_logprob_denoising_std: 0.1
#
network_path: ${base_policy_path}
actor:
_target_: model.diffusion.mlp_diffusion.DiffusionMLP
time_dim: 16
mlp_dims: [512, 512, 512]
activation_type: ReLU
residual_style: True
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
horizon_steps: ${horizon_steps}
transition_dim: ${transition_dim}
critic:
_target_: model.common.critic.CriticObs
obs_dim: ${obs_dim}
mlp_dims: [256, 256, 256]
activation_type: Mish
residual_style: True
ft_denoising_steps: ${ft_denoising_steps}
transition_dim: ${transition_dim}
horizon_steps: ${horizon_steps}
obs_dim: ${obs_dim}
action_dim: ${action_dim}
cond_steps: ${cond_steps}
denoising_steps: ${denoising_steps}
device: ${device}
cfg/d3il/finetune/avoid_m2/ft_ppo_gaussian_mlp.yaml
+105
View File
@@ -0,0 +1,105 @@
defaults:
- _self_
hydra:
run:
dir: ${logdir}
_target_: agent.finetune.train_ppo_gaussian_agent.TrainPPOGaussianAgent
name: ${env_name}_m2_gaussian_mlp_ta${horizon_steps}
logdir: ${oc.env:DPPO_LOG_DIR}/d3il-finetune/${name}/${now:%Y-%m-%d}_${now:%H-%M-%S}_${seed}
base_policy_path: ${oc.env:DPPO_LOG_DIR}/d3il-pretrain/m2/avoid_d57_r12_pre_gaussian_mlp_ta4/2024-07-07_02-15-50/checkpoint/state_10000.pt
normalization_path: ${oc.env:DPPO_DATA_DIR}/d3il/avoid_m2/normalization.npz
seed: 42
device: cuda:0
env_name: avoiding-m5
mode: d57_r12 # M2, desired modes 5 and 7, required modes 1 and 2
obs_dim: 4
action_dim: 2
transition_dim: ${action_dim}
cond_steps: 1
horizon_steps: 4
act_steps: 4
env:
n_envs: 50
name: ${env_name}
max_episode_steps: 100
reset_at_iteration: True
save_video: False
best_reward_threshold_for_success: 2
save_full_observations: True
wrappers:
d3il_lowdim:
normalization_path: ${normalization_path}
multi_step:
n_obs_steps: ${cond_steps}
n_action_steps: ${act_steps}
max_episode_steps: ${env.max_episode_steps}
pass_full_observations: ${env.save_full_observations}
reset_within_step: False
wandb:
entity: ${oc.env:DPPO_WANDB_ENTITY}
project: d3il-${env_name}-m2-finetune
run: ${now:%H-%M-%S}_${name}
train:
n_train_itr: 51
n_critic_warmup_itr: 1
n_steps: ${eval:'round(${env.max_episode_steps} / ${act_steps})'}
gamma: 0.99
actor_lr: 1e-5
actor_weight_decay: 0
actor_lr_scheduler:
first_cycle_steps: 100
warmup_steps: 10
min_lr: 1e-5
critic_lr: 1e-3
critic_weight_decay: 0
critic_lr_scheduler:
first_cycle_steps: 100
warmup_steps: 10
min_lr: 1e-3
save_model_freq: 100
val_freq: 100 # no eval, always train mode
force_train: True
render:
freq: 1
num: 10
plotter:
_target_: env.plot_traj.TrajPlotter
env_type: avoid
normalization_path: ${normalization_path}
# PPO specific
reward_scale_running: True
reward_scale_const: 1.0
gae_lambda: 0.95
batch_size: ${eval:'round(${train.n_steps} * ${env.n_envs} / 2)'}
update_epochs: 10
vf_coef: 0.5
target_kl: 1
model:
_target_: model.rl.gaussian_ppo.PPO_Gaussian
clip_ploss_coef: 0.1
randn_clip_value: 3
network_path: ${base_policy_path}
actor:
_target_: model.common.mlp_gaussian.Gaussian_MLP
mlp_dims: [256, 256, 256] # smaller MLP for less overfitting
activation_type: ReLU
residual_style: True
fixed_std: 0.1
learn_fixed_std: False
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
horizon_steps: ${horizon_steps}
transition_dim: ${transition_dim}
critic:
_target_: model.common.critic.CriticObs
mlp_dims: [256, 256, 256]
residual_style: True
obs_dim: ${obs_dim}
horizon_steps: ${horizon_steps}
cond_steps: ${cond_steps}
device: ${device}
cfg/d3il/finetune/avoid_m2/ft_ppo_gmm_mlp.yaml
+106
View File
@@ -0,0 +1,106 @@
defaults:
- _self_
hydra:
run:
dir: ${logdir}
_target_: agent.finetune.train_ppo_gaussian_agent.TrainPPOGaussianAgent
name: ${env_name}_m2_gmm_mlp_ta${horizon_steps}
logdir: ${oc.env:DPPO_LOG_DIR}/d3il-finetune/${name}/${now:%Y-%m-%d}_${now:%H-%M-%S}_${seed}
base_policy_path: ${oc.env:DPPO_LOG_DIR}/d3il-pretrain/m2/avoid_d57_r12_pre_gmm_mlp_ta4/2024-07-10_15-44-32/checkpoint/state_10000.pt
normalization_path: ${oc.env:DPPO_DATA_DIR}/d3il/avoid_m2/normalization.npz
seed: 42
device: cuda:0
env_name: avoiding-m5
mode: d57_r12 # M2, desired modes 5 and 7, required modes 1 and 2
obs_dim: 4
action_dim: 2
transition_dim: ${action_dim}
cond_steps: 1
horizon_steps: 4
act_steps: 4
num_modes: 5
env:
n_envs: 50
name: ${env_name}
max_episode_steps: 100
reset_at_iteration: True
save_video: False
best_reward_threshold_for_success: 2
save_full_observations: True
wrappers:
d3il_lowdim:
normalization_path: ${normalization_path}
multi_step:
n_obs_steps: ${cond_steps}
n_action_steps: ${act_steps}
max_episode_steps: ${env.max_episode_steps}
pass_full_observations: ${env.save_full_observations}
reset_within_step: False
wandb:
entity: ${oc.env:DPPO_WANDB_ENTITY}
project: d3il-${env_name}-m2-finetune
run: ${now:%H-%M-%S}_${name}
train:
n_train_itr: 51
n_critic_warmup_itr: 1
n_steps: ${eval:'round(${env.max_episode_steps} / ${act_steps})'}
gamma: 0.99
actor_lr: 1e-5
actor_weight_decay: 0
actor_lr_scheduler:
first_cycle_steps: 100
warmup_steps: 10
min_lr: 1e-5
critic_lr: 1e-3
critic_weight_decay: 0
critic_lr_scheduler:
first_cycle_steps: 100
warmup_steps: 10
min_lr: 1e-3
save_model_freq: 100
val_freq: 100 # no eval, always train mode
force_train: True
render:
freq: 1
num: 10
plotter:
_target_: env.plot_traj.TrajPlotter
env_type: avoid
normalization_path: ${normalization_path}
# PPO specific
reward_scale_running: True
reward_scale_const: 1.0
gae_lambda: 0.95
batch_size: ${eval:'round(${train.n_steps} * ${env.n_envs} / 2)'}
update_epochs: 10
vf_coef: 0.5
target_kl: 1
model:
_target_: model.rl.gmm_ppo.PPO_GMM
clip_ploss_coef: 0.1
network_path: ${base_policy_path}
actor:
_target_: model.common.mlp_gmm.GMM_MLP
mlp_dims: [256, 256] # smaller MLP for less overfitting
activation_type: ReLU
residual_style: False
fixed_std: 0.1
learn_fixed_std: False
num_modes: ${num_modes}
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
horizon_steps: ${horizon_steps}
transition_dim: ${transition_dim}
critic:
_target_: model.common.critic.CriticObs
mlp_dims: [256, 256, 256]
residual_style: True
obs_dim: ${obs_dim}
horizon_steps: ${horizon_steps}
cond_steps: ${cond_steps}
device: ${device}
cfg/d3il/finetune/avoid_m3/ft_ppo_diffusion_mlp.yaml
+119
View File
@@ -0,0 +1,119 @@
defaults:
- _self_
hydra:
run:
dir: ${logdir}
_target_: agent.finetune.train_ppo_diffusion_agent.TrainPPODiffusionAgent
name: ${env_name}_m3_ft_diffusion_mlp_ta${horizon_steps}_td${denoising_steps}_tdf${ft_denoising_steps}
logdir: ${oc.env:DPPO_LOG_DIR}/d3il-finetune/${name}/${now:%Y-%m-%d}_${now:%H-%M-%S}_${seed}
base_policy_path: ${oc.env:DPPO_LOG_DIR}/d3il-pretrain/m3/avoid_d58_r12_pre_diffusion_mlp_ta4_td20/2024-07-07_13-54-54/checkpoint/state_15000.pt
normalization_path: ${oc.env:DPPO_DATA_DIR}/d3il/avoid_m3/normalization.npz
seed: 42
device: cuda:0
env_name: avoiding-m5
mode: d58_r12 # M3, desired modes 5 and 8, required modes 1 and 2
obs_dim: 4
action_dim: 2
transition_dim: ${action_dim}
denoising_steps: 20
ft_denoising_steps: 10
cond_steps: 1
horizon_steps: 4
act_steps: 4
env:
n_envs: 50
name: ${env_name}
max_episode_steps: 100
reset_at_iteration: True
save_video: False
best_reward_threshold_for_success: 2
save_full_observations: True
wrappers:
d3il_lowdim:
normalization_path: ${normalization_path}
multi_step:
n_obs_steps: ${cond_steps}
n_action_steps: ${act_steps}
max_episode_steps: ${env.max_episode_steps}
pass_full_observations: ${env.save_full_observations}
reset_within_step: False
wandb:
entity: ${oc.env:DPPO_WANDB_ENTITY}
project: d3il-${env_name}-m3-finetune
run: ${now:%H-%M-%S}_${name}
train:
n_train_itr: 51
n_critic_warmup_itr: 1
n_steps: ${eval:'round(${env.max_episode_steps} / ${act_steps})'}
gamma: 0.99
actor_lr: 1e-5
actor_weight_decay: 0
actor_lr_scheduler:
first_cycle_steps: 100
warmup_steps: 10
min_lr: 1e-5
critic_lr: 1e-3
critic_weight_decay: 0
critic_lr_scheduler:
first_cycle_steps: 100
warmup_steps: 10
min_lr: 1e-3
save_model_freq: 100
val_freq: 100 # no eval, always train mode
force_train: True
render:
freq: 1
num: 10
plotter:
_target_: env.plot_traj.TrajPlotter
env_type: avoid
normalization_path: ${normalization_path}
# PPO specific
reward_scale_running: True
reward_scale_const: 1.0
gae_lambda: 0.95
batch_size: ${eval:'round(${train.n_steps} * ${env.n_envs} / 2)'}
update_epochs: 10
vf_coef: 0.5
target_kl: 1
model:
_target_: model.diffusion.diffusion_ppo.PPODiffusion
# HP to tune
gamma_denoising: 0.95
clip_ploss_coef: 0.1
clip_ploss_coef_base: 0.1
clip_ploss_coef_rate: 1
randn_clip_value: 3
min_sampling_denoising_std: 0.1
min_logprob_denoising_std: 0.1
#
network_path: ${base_policy_path}
actor:
_target_: model.diffusion.mlp_diffusion.DiffusionMLP
time_dim: 16
mlp_dims: [512, 512, 512]
activation_type: ReLU
residual_style: True
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
horizon_steps: ${horizon_steps}
transition_dim: ${transition_dim}
critic:
_target_: model.common.critic.CriticObs
obs_dim: ${obs_dim}
mlp_dims: [256, 256, 256]
activation_type: Mish
residual_style: True
ft_denoising_steps: ${ft_denoising_steps}
transition_dim: ${transition_dim}
horizon_steps: ${horizon_steps}
obs_dim: ${obs_dim}
action_dim: ${action_dim}
cond_steps: ${cond_steps}
denoising_steps: ${denoising_steps}
device: ${device}
cfg/d3il/finetune/avoid_m3/ft_ppo_gaussian_mlp.yaml
+105
View File
@@ -0,0 +1,105 @@
defaults:
- _self_
hydra:
run:
dir: ${logdir}
_target_: agent.finetune.train_ppo_gaussian_agent.TrainPPOGaussianAgent
name: ${env_name}_m3_gaussian_mlp_ta${horizon_steps}
logdir: ${oc.env:DPPO_LOG_DIR}/d3il-finetune/${name}/${now:%Y-%m-%d}_${now:%H-%M-%S}_${seed}
base_policy_path: ${oc.env:DPPO_LOG_DIR}/d3il-pretrain/m3/avoid_d58_r12_pre_gaussian_mlp_ta4/2024-07-07_13-11-49/checkpoint/state_10000.pt
normalization_path: ${oc.env:DPPO_DATA_DIR}/d3il/avoid_m3/normalization.npz
seed: 42
device: cuda:0
env_name: avoiding-m5
mode: d58_r12 # M3, desired modes 5 and 8, required modes 1 and 2
obs_dim: 4
action_dim: 2
transition_dim: ${action_dim}
cond_steps: 1
horizon_steps: 4
act_steps: 4
env:
n_envs: 50
name: ${env_name}
max_episode_steps: 100
reset_at_iteration: True
save_video: False
best_reward_threshold_for_success: 2
save_full_observations: True
wrappers:
d3il_lowdim:
normalization_path: ${normalization_path}
multi_step:
n_obs_steps: ${cond_steps}
n_action_steps: ${act_steps}
max_episode_steps: ${env.max_episode_steps}
pass_full_observations: ${env.save_full_observations}
reset_within_step: False
wandb:
entity: ${oc.env:DPPO_WANDB_ENTITY}
project: d3il-${env_name}-m3-finetune
run: ${now:%H-%M-%S}_${name}
train:
n_train_itr: 51
n_critic_warmup_itr: 1
n_steps: ${eval:'round(${env.max_episode_steps} / ${act_steps})'}
gamma: 0.99
actor_lr: 1e-5
actor_weight_decay: 0
actor_lr_scheduler:
first_cycle_steps: 100
warmup_steps: 10
min_lr: 1e-5
critic_lr: 1e-3
critic_weight_decay: 0
critic_lr_scheduler:
first_cycle_steps: 100
warmup_steps: 10
min_lr: 1e-3
save_model_freq: 100
val_freq: 100 # no eval, always train mode
force_train: True
render:
freq: 1
num: 10
plotter:
_target_: env.plot_traj.TrajPlotter
env_type: avoid
normalization_path: ${normalization_path}
# PPO specific
reward_scale_running: True
reward_scale_const: 1.0
gae_lambda: 0.95
batch_size: ${eval:'round(${train.n_steps} * ${env.n_envs} / 2)'}
update_epochs: 10
vf_coef: 0.5
target_kl: 1
model:
_target_: model.rl.gaussian_ppo.PPO_Gaussian
clip_ploss_coef: 0.1
randn_clip_value: 3
network_path: ${base_policy_path}
actor:
_target_: model.common.mlp_gaussian.Gaussian_MLP
mlp_dims: [256, 256, 256] # smaller MLP for less overfitting
activation_type: ReLU
residual_style: True
fixed_std: 0.1
learn_fixed_std: False
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
horizon_steps: ${horizon_steps}
transition_dim: ${transition_dim}
critic:
_target_: model.common.critic.CriticObs
mlp_dims: [256, 256, 256]
residual_style: True
obs_dim: ${obs_dim}
horizon_steps: ${horizon_steps}
cond_steps: ${cond_steps}
device: ${device}
cfg/d3il/finetune/avoid_m3/ft_ppo_gmm_mlp.yaml
+106
View File
@@ -0,0 +1,106 @@
defaults:
- _self_
hydra:
run:
dir: ${logdir}
_target_: agent.finetune.train_ppo_gaussian_agent.TrainPPOGaussianAgent
name: ${env_name}_m3_gmm_mlp_ta${horizon_steps}
logdir: ${oc.env:DPPO_LOG_DIR}/d3il-finetune/${name}/${now:%Y-%m-%d}_${now:%H-%M-%S}_${seed}
base_policy_path: ${oc.env:DPPO_LOG_DIR}/d3il-pretrain/m3/avoid_d58_r12_pre_gmm_mlp_ta4/2024-07-10_17-01-50/checkpoint/state_10000.pt
normalization_path: ${oc.env:DPPO_DATA_DIR}/d3il/avoid_m3/normalization.npz
seed: 42
device: cuda:0
env_name: avoiding-m5
mode: d58_r12 # M3, desired modes 5 and 8, required modes 1 and 2
obs_dim: 4
action_dim: 2
transition_dim: ${action_dim}
cond_steps: 1
horizon_steps: 4
act_steps: 4
num_modes: 5
env:
n_envs: 50
name: ${env_name}
max_episode_steps: 100
reset_at_iteration: True
save_video: False
best_reward_threshold_for_success: 2
save_full_observations: True
wrappers:
d3il_lowdim:
normalization_path: ${normalization_path}
multi_step:
n_obs_steps: ${cond_steps}
n_action_steps: ${act_steps}
max_episode_steps: ${env.max_episode_steps}
pass_full_observations: ${env.save_full_observations}
reset_within_step: False
wandb:
entity: ${oc.env:DPPO_WANDB_ENTITY}
project: d3il-${env_name}-m3-finetune
run: ${now:%H-%M-%S}_${name}
train:
n_train_itr: 51
n_critic_warmup_itr: 1
n_steps: ${eval:'round(${env.max_episode_steps} / ${act_steps})'}
gamma: 0.99
actor_lr: 1e-5
actor_weight_decay: 0
actor_lr_scheduler:
first_cycle_steps: 100
warmup_steps: 10
min_lr: 1e-5
critic_lr: 1e-3
critic_weight_decay: 0
critic_lr_scheduler:
first_cycle_steps: 100
warmup_steps: 10
min_lr: 1e-3
save_model_freq: 100
val_freq: 100 # no eval, always train mode
force_train: True
render:
freq: 1
num: 10
plotter:
_target_: env.plot_traj.TrajPlotter
env_type: avoid
normalization_path: ${normalization_path}
# PPO specific
reward_scale_running: True
reward_scale_const: 1.0
gae_lambda: 0.95
batch_size: ${eval:'round(${train.n_steps} * ${env.n_envs} / 2)'}
update_epochs: 10
vf_coef: 0.5
target_kl: 1
model:
_target_: model.rl.gmm_ppo.PPO_GMM
clip_ploss_coef: 0.1
network_path: ${base_policy_path}
actor:
_target_: model.common.mlp_gmm.GMM_MLP
mlp_dims: [256, 256] # smaller MLP for less overfitting
activation_type: ReLU
residual_style: False
fixed_std: 0.1
learn_fixed_std: False
num_modes: ${num_modes}
cond_dim: ${eval:'${obs_dim} * ${cond_steps}'}
horizon_steps: ${horizon_steps}
transition_dim: ${transition_dim}
critic:
_target_: model.common.critic.CriticObs
mlp_dims: [256, 256, 256]
residual_style: True
obs_dim: ${obs_dim}
horizon_steps: ${horizon_steps}
cond_steps: ${cond_steps}
device: ${device}