dodox/fancy_gym
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge pull request #75 from D-o-d-o-x/great_refactor

Browse Source 
Refactor and Upgrade to Gymnasium
This commit is contained in:
Dominik Roth
2023-10-11 13:42:00 +02:00
committed by GitHub
parent 5ad2942e9a a04a4b6e67
commit c420a96d4f
84 changed files with 3094 additions and 2741 deletions
Download Patch File Download Diff File Expand all files Collapse all files
fancy_gym/examples/example_replanning_envs.py
+13 -10
View File
@@ -1,20 +1,23 @@
import gymnasium as gym
import fancy_gym
def example_run_replanning_env(env_name="BoxPushingDenseReplanProDMP-v0", seed=1, iterations=1, render=False):
env = fancy_gym.make(env_name, seed=seed)
env.reset()
def example_run_replanning_env(env_name="fancy_ProDMP/BoxPushingDenseReplan-v0", seed=1, iterations=1, render=False):
env = gym.make(env_name)
env.reset(seed=seed)
for i in range(iterations):
done = False
while done is False:
ac = env.action_space.sample()
obs, reward, done, info = env.step(ac)
obs, reward, terminated, truncated, info = env.step(ac)
if render:
env.render(mode="human")
if done:
if terminated or truncated:
env.reset()
env.close()
del env
def example_custom_replanning_envs(seed=0, iteration=100, render=True):
# id for a step-based environment
base_env_id = "BoxPushingDense-v0"
@@ -22,7 +25,7 @@ def example_custom_replanning_envs(seed=0, iteration=100, render=True):
wrappers = [fancy_gym.envs.mujoco.box_pushing.mp_wrapper.MPWrapper]
trajectory_generator_kwargs = {'trajectory_generator_type': 'prodmp',
'weight_scale': 1}
'weights_scale': 1}
phase_generator_kwargs = {'phase_generator_type': 'exp'}
controller_kwargs = {'controller_type': 'velocity'}
basis_generator_kwargs = {'basis_generator_type': 'prodmp',
@@ -46,8 +49,8 @@ def example_custom_replanning_envs(seed=0, iteration=100, render=True):
for i in range(iteration):
ac = env.action_space.sample()
obs, reward, done, info = env.step(ac)
if done:
obs, reward, terminated, truncated, info = env.step(ac)
if terminated or truncated:
env.reset()
env.close()
@@ -56,7 +59,7 @@ def example_custom_replanning_envs(seed=0, iteration=100, render=True):
if __name__ == "__main__":
# run a registered replanning environment
example_run_replanning_env(env_name="BoxPushingDenseReplanProDMP-v0", seed=1, iterations=1, render=False)
example_run_replanning_env(env_name="fancy_ProDMP/BoxPushingDenseReplan-v0", seed=1, iterations=1, render=False)
# run a custom replanning environment
example_custom_replanning_envs(seed=0, iteration=8, render=True)
example_custom_replanning_envs(seed=0, iteration=8, render=True)
fancy_gym/examples/examples_dmc.py
+14 -13
View File
@@ -1,7 +1,8 @@
import gymnasium as gym
import fancy_gym
def example_dmc(env_id="dmc:fish-swim", seed=1, iterations=1000, render=True):
def example_dmc(env_id="dm_control/fish-swim", seed=1, iterations=1000, render=True):
"""
Example for running a DMC based env in the step based setting.
The env_id has to be specified as `domain_name:task_name` or
@@ -16,9 +17,9 @@ def example_dmc(env_id="dmc:fish-swim", seed=1, iterations=1000, render=True):
Returns:
"""
env = fancy_gym.make(env_id, seed)
env = gym.make(env_id)
rewards = 0
obs = env.reset()
obs = env.reset(seed=seed)
print("observation shape:", env.observation_space.shape)
print("action shape:", env.action_space.shape)
@@ -26,10 +27,10 @@ def example_dmc(env_id="dmc:fish-swim", seed=1, iterations=1000, render=True):
ac = env.action_space.sample()
if render:
env.render(mode="human")
obs, reward, done, info = env.step(ac)
obs, reward, terminated, truncated, info = env.step(ac)
rewards += reward
if done:
if terminated or truncated:
print(env_id, rewards)
rewards = 0
obs = env.reset()
@@ -56,7 +57,7 @@ def example_custom_dmc_and_mp(seed=1, iterations=1, render=True):
"""
# Base DMC name, according to structure of above example
base_env_id = "dmc:ball_in_cup-catch"
base_env_id = "dm_control/ball_in_cup-catch"
# Replace this wrapper with the custom wrapper for your environment by inheriting from the RawInterfaceWrapper.
# You can also add other gym.Wrappers in case they are needed.
@@ -65,8 +66,8 @@ def example_custom_dmc_and_mp(seed=1, iterations=1, render=True):
trajectory_generator_kwargs = {'trajectory_generator_type': 'promp'}
phase_generator_kwargs = {'phase_generator_type': 'linear'}
controller_kwargs = {'controller_type': 'motor',
"p_gains": 1.0,
"d_gains": 0.1,}
"p_gains": 1.0,
"d_gains": 0.1, }
basis_generator_kwargs = {'basis_generator_type': 'zero_rbf',
'num_basis': 5,
'num_basis_zero_start': 1
@@ -102,10 +103,10 @@ def example_custom_dmc_and_mp(seed=1, iterations=1, render=True):
# number of samples/full trajectories (multiple environment steps)
for i in range(iterations):
ac = env.action_space.sample()
obs, reward, done, info = env.step(ac)
obs, reward, terminated, truncated, info = env.step(ac)
rewards += reward
if done:
if terminated or truncated:
print(base_env_id, rewards)
rewards = 0
obs = env.reset()
@@ -123,14 +124,14 @@ if __name__ == '__main__':
render = True
# # Standard DMC Suite tasks
example_dmc("dmc:fish-swim", seed=10, iterations=1000, render=render)
example_dmc("dm_control/fish-swim", seed=10, iterations=1000, render=render)
#
# # Manipulation tasks
# # Disclaimer: The vision versions are currently not integrated and yield an error
example_dmc("dmc:manipulation-reach_site_features", seed=10, iterations=250, render=render)
example_dmc("dm_control/manipulation-reach_site_features", seed=10, iterations=250, render=render)
#
# # Gym + DMC hybrid task provided in the MP framework
example_dmc("dmc_ball_in_cup-catch_promp-v0", seed=10, iterations=1, render=render)
example_dmc("dm_control_ProMP/ball_in_cup-catch-v0", seed=10, iterations=1, render=render)
# Custom DMC task # Different seed, because the episode is longer for this example and the name+seed combo is
# already registered above
fancy_gym/examples/examples_general.py
+12 -10
View File
@@ -1,6 +1,6 @@
from collections import defaultdict
import gym
import gymnasium as gym
import numpy as np
import fancy_gym
@@ -21,27 +21,27 @@ def example_general(env_id="Pendulum-v1", seed=1, iterations=1000, render=True):
"""
env = fancy_gym.make(env_id, seed)
env = gym.make(env_id)
rewards = 0
obs = env.reset()
obs = env.reset(seed=seed)
print("Observation shape: ", env.observation_space.shape)
print("Action shape: ", env.action_space.shape)
# number of environment steps
for i in range(iterations):
obs, reward, done, info = env.step(env.action_space.sample())
obs, reward, terminated, truncated, info = env.step(env.action_space.sample())
rewards += reward
if render:
env.render()
if done:
if terminated or truncated:
print(rewards)
rewards = 0
obs = env.reset()
def example_async(env_id="HoleReacher-v0", n_cpu=4, seed=int('533D', 16), n_samples=800):
def example_async(env_id="fancy/HoleReacher-v0", n_cpu=4, seed=int('533D', 16), n_samples=800):
"""
Example for running any env in a vectorized multiprocessing setting to generate more samples faster.
This also includes DMC and DMP environments when leveraging our custom make_env function.
@@ -69,12 +69,15 @@ def example_async(env_id="HoleReacher-v0", n_cpu=4, seed=int('533D', 16), n_samp
# this would generate more samples than requested if n_samples % num_envs != 0
repeat = int(np.ceil(n_samples / env.num_envs))
for i in range(repeat):
obs, reward, done, info = env.step(env.action_space.sample())
obs, reward, terminated, truncated, info = env.step(env.action_space.sample())
buffer['obs'].append(obs)
buffer['reward'].append(reward)
buffer['done'].append(done)
buffer['terminated'].append(terminated)
buffer['truncated'].append(truncated)
buffer['info'].append(info)
rewards += reward
done = terminated or truncated
if np.any(done):
print(f"Reward at iteration {i}: {rewards[done]}")
rewards[done] = 0
@@ -90,11 +93,10 @@ if __name__ == '__main__':
example_general("Pendulum-v1", seed=10, iterations=200, render=render)
# Mujoco task from framework
example_general("Reacher5d-v0", seed=10, iterations=200, render=render)
example_general("fancy/Reacher5d-v0", seed=10, iterations=200, render=render)
# # OpenAI Mujoco task
example_general("HalfCheetah-v2", seed=10, render=render)
# Vectorized multiprocessing environments
# example_async(env_id="HoleReacher-v0", n_cpu=2, seed=int('533D', 16), n_samples=2 * 200)
fancy_gym/examples/examples_metaworld.py
+13 -13
View File
@@ -1,7 +1,8 @@
import gymnasium as gym
import fancy_gym
def example_dmc(env_id="fish-swim", seed=1, iterations=1000, render=True):
def example_meta(env_id="fish-swim", seed=1, iterations=1000, render=True):
"""
Example for running a MetaWorld based env in the step based setting.
The env_id has to be specified as `task_name-v2`. V1 versions are not supported and we always
@@ -17,9 +18,9 @@ def example_dmc(env_id="fish-swim", seed=1, iterations=1000, render=True):
Returns:
"""
env = fancy_gym.make(env_id, seed)
env = gym.make(env_id)
rewards = 0
obs = env.reset()
obs = env.reset(seed=seed)
print("observation shape:", env.observation_space.shape)
print("action shape:", env.action_space.shape)
@@ -29,9 +30,9 @@ def example_dmc(env_id="fish-swim", seed=1, iterations=1000, render=True):
# THIS NEEDS TO BE SET TO FALSE FOR NOW, BECAUSE THE INTERFACE FOR RENDERING IS DIFFERENT TO BASIC GYM
# TODO: Remove this, when Metaworld fixes its interface.
env.render(False)
obs, reward, done, info = env.step(ac)
obs, reward, terminated, truncated, info = env.step(ac)
rewards += reward
if done:
if terminated or truncated:
print(env_id, rewards)
rewards = 0
obs = env.reset()
@@ -40,7 +41,7 @@ def example_dmc(env_id="fish-swim", seed=1, iterations=1000, render=True):
del env
def example_custom_dmc_and_mp(seed=1, iterations=1, render=True):
def example_custom_meta_and_mp(seed=1, iterations=1, render=True):
"""
Example for running a custom movement primitive based environments.
Our already registered environments follow the same structure.
@@ -58,7 +59,7 @@ def example_custom_dmc_and_mp(seed=1, iterations=1, render=True):
"""
# Base MetaWorld name, according to structure of above example
base_env_id = "metaworld:button-press-v2"
base_env_id = "metaworld/button-press-v2"
# Replace this wrapper with the custom wrapper for your environment by inheriting from the RawInterfaceWrapper.
# You can also add other gym.Wrappers in case they are needed.
@@ -103,10 +104,10 @@ def example_custom_dmc_and_mp(seed=1, iterations=1, render=True):
# number of samples/full trajectories (multiple environment steps)
for i in range(iterations):
ac = env.action_space.sample()
obs, reward, done, info = env.step(ac)
obs, reward, terminated, truncated, info = env.step(ac)
rewards += reward
if done:
if terminated or truncated:
print(base_env_id, rewards)
rewards = 0
obs = env.reset()
@@ -124,11 +125,10 @@ if __name__ == '__main__':
render = False
# # Standard Meta world tasks
example_dmc("metaworld:button-press-v2", seed=10, iterations=500, render=render)
example_meta("metaworld/button-press-v2", seed=10, iterations=500, render=render)
# # MP + MetaWorld hybrid task provided in the our framework
example_dmc("ButtonPressProMP-v2", seed=10, iterations=1, render=render)
example_meta("metaworld_ProMP/ButtonPress-v2", seed=10, iterations=1, render=render)
#
# # Custom MetaWorld task
example_custom_dmc_and_mp(seed=10, iterations=1, render=render)
example_custom_meta_and_mp(seed=10, iterations=1, render=render)
fancy_gym/examples/examples_movement_primitives.py
+24 -26
View File
@@ -1,7 +1,8 @@
import gymnasium as gym
import fancy_gym
def example_mp(env_name="HoleReacherProMP-v0", seed=1, iterations=1, render=True):
def example_mp(env_name="fancy_ProMP/HoleReacher-v0", seed=1, iterations=1, render=True):
"""
Example for running a black box based environment, which is already registered
Args:
@@ -15,11 +16,11 @@ def example_mp(env_name="HoleReacherProMP-v0", seed=1, iterations=1, render=True
"""
# Equivalent to gym, we have a make function which can be used to create environments.
# It takes care of seeding and enables the use of a variety of external environments using the gym interface.
env = fancy_gym.make(env_name, seed)
env = gym.make(env_name)
returns = 0
# env.render(mode=None)
obs = env.reset()
obs = env.reset(seed=seed)
# number of samples/full trajectories (multiple environment steps)
for i in range(iterations):
@@ -41,16 +42,16 @@ def example_mp(env_name="HoleReacherProMP-v0", seed=1, iterations=1, render=True
# This executes a full trajectory and gives back the context (obs) of the last step in the trajectory, or the
# full observation space of the last step, if replanning/sub-trajectory learning is used. The 'reward' is equal
# to the return of a trajectory. Default is the sum over the step-wise rewards.
obs, reward, done, info = env.step(ac)
obs, reward, terminated, truncated, info = env.step(ac)
# Aggregated returns
returns += reward
if done:
if terminated or truncated:
print(reward)
obs = env.reset()
def example_custom_mp(env_name="Reacher5dProMP-v0", seed=1, iterations=1, render=True):
def example_custom_mp(env_name="fancy_ProMP/Reacher5d-v0", seed=1, iterations=1, render=True):
"""
Example for running a movement primitive based environment, which is already registered
Args:
@@ -62,12 +63,9 @@ def example_custom_mp(env_name="Reacher5dProMP-v0", seed=1, iterations=1, render
Returns:
"""
# Changing the arguments of the black box env is possible by providing them to gym as with all kwargs.
# Changing the arguments of the black box env is possible by providing them to gym through mp_config_override.
# E.g. here for way to many basis functions
env = fancy_gym.make(env_name, seed, basis_generator_kwargs={'num_basis': 1000})
# env = fancy_gym.make(env_name, seed)
# mp_dict.update({'black_box_kwargs': {'learn_sub_trajectories': True}})
# mp_dict.update({'black_box_kwargs': {'do_replanning': lambda pos, vel, t: lambda t: t % 100}})
env = gym.make(env_name, seed, mp_config_override={'basis_generator_kwargs': {'num_basis': 1000}})
returns = 0
obs = env.reset()
@@ -79,10 +77,10 @@ def example_custom_mp(env_name="Reacher5dProMP-v0", seed=1, iterations=1, render
# number of samples/full trajectories (multiple environment steps)
for i in range(iterations):
ac = env.action_space.sample()
obs, reward, done, info = env.step(ac)
obs, reward, terminated, truncated, info = env.step(ac)
returns += reward
if done:
if terminated or truncated:
print(i, reward)
obs = env.reset()
@@ -106,7 +104,7 @@ def example_fully_custom_mp(seed=1, iterations=1, render=True):
"""
base_env_id = "Reacher5d-v0"
base_env_id = "fancy/Reacher5d-v0"
# Replace this wrapper with the custom wrapper for your environment by inheriting from the RawInterfaceWrapper.
# You can also add other gym.Wrappers in case they are needed.
@@ -114,7 +112,7 @@ def example_fully_custom_mp(seed=1, iterations=1, render=True):
# For a ProMP
trajectory_generator_kwargs = {'trajectory_generator_type': 'promp',
'weight_scale': 2}
'weights_scale': 2}
phase_generator_kwargs = {'phase_generator_type': 'linear'}
controller_kwargs = {'controller_type': 'velocity'}
basis_generator_kwargs = {'basis_generator_type': 'zero_rbf',
@@ -124,7 +122,7 @@ def example_fully_custom_mp(seed=1, iterations=1, render=True):
# # For a DMP
# trajectory_generator_kwargs = {'trajectory_generator_type': 'dmp',
# 'weight_scale': 500}
# 'weights_scale': 500}
# phase_generator_kwargs = {'phase_generator_type': 'exp',
# 'alpha_phase': 2.5}
# controller_kwargs = {'controller_type': 'velocity'}
@@ -145,10 +143,10 @@ def example_fully_custom_mp(seed=1, iterations=1, render=True):
# number of samples/full trajectories (multiple environment steps)
for i in range(iterations):
ac = env.action_space.sample()
obs, reward, done, info = env.step(ac)
obs, reward, terminated, truncated, info = env.step(ac)
rewards += reward
if done:
if terminated or truncated:
print(rewards)
rewards = 0
obs = env.reset()
@@ -157,20 +155,20 @@ def example_fully_custom_mp(seed=1, iterations=1, render=True):
if __name__ == '__main__':
render = False
# DMP
example_mp("HoleReacherDMP-v0", seed=10, iterations=5, render=render)
example_mp("fancy_DMP/HoleReacher-v0", seed=10, iterations=5, render=render)
# ProMP
example_mp("HoleReacherProMP-v0", seed=10, iterations=5, render=render)
example_mp("BoxPushingTemporalSparseProMP-v0", seed=10, iterations=1, render=render)
example_mp("TableTennis4DProMP-v0", seed=10, iterations=20, render=render)
example_mp("fancy_ProMP/HoleReacher-v0", seed=10, iterations=5, render=render)
example_mp("fancy_ProMP/BoxPushingTemporalSparse-v0", seed=10, iterations=1, render=render)
example_mp("fancy_ProMP/TableTennis4D-v0", seed=10, iterations=20, render=render)
# ProDMP with Replanning
example_mp("BoxPushingDenseReplanProDMP-v0", seed=10, iterations=4, render=render)
example_mp("TableTennis4DReplanProDMP-v0", seed=10, iterations=20, render=render)
example_mp("TableTennisWindReplanProDMP-v0", seed=10, iterations=20, render=render)
example_mp("fancy_ProDMP/BoxPushingDenseReplan-v0", seed=10, iterations=4, render=render)
example_mp("fancy_ProDMP/TableTennis4DReplan-v0", seed=10, iterations=20, render=render)
example_mp("fancy_ProDMP/TableTennisWindReplan-v0", seed=10, iterations=20, render=render)
# Altered basis functions
obs1 = example_custom_mp("Reacher5dProMP-v0", seed=10, iterations=1, render=render)
obs1 = example_custom_mp("fancy_ProMP/Reacher5d-v0", seed=10, iterations=1, render=render)
# Custom MP
example_fully_custom_mp(seed=10, iterations=1, render=render)
fancy_gym/examples/examples_open_ai.py
+6 -7
View File
@@ -1,3 +1,4 @@
import gymnasium as gym
import fancy_gym
@@ -12,11 +13,10 @@ def example_mp(env_name, seed=1, render=True):
Returns:
"""
# While in this case gym.make() is possible to use as well, we recommend our custom make env function.
env = fancy_gym.make(env_name, seed)
env = gym.make(env_name)
returns = 0
obs = env.reset()
obs = env.reset(seed=seed)
# number of samples/full trajectories (multiple environment steps)
for i in range(10):
if render and i % 2 == 0:
@@ -24,14 +24,13 @@ def example_mp(env_name, seed=1, render=True):
else:
env.render()
ac = env.action_space.sample()
obs, reward, done, info = env.step(ac)
obs, reward, terminated, truncated, info = env.step(ac)
returns += reward
if done:
if terminated or truncated:
print(returns)
obs = env.reset()
if __name__ == '__main__':
example_mp("ReacherProMP-v2")
example_mp("gym_ProMP/Reacher-v2")
fancy_gym/examples/mp_params_tuning.py
+7 -3
View File
@@ -1,10 +1,14 @@
import gymnasium as gym
import fancy_gym
def compare_bases_shape(env1_id, env2_id):
env1 = fancy_gym.make(env1_id, seed=0)
env1 = gym.make(env1_id)
env1.traj_gen.show_scaled_basis(plot=True)
env2 = fancy_gym.make(env2_id, seed=0)
env2 = gym.make(env2_id)
env2.traj_gen.show_scaled_basis(plot=True)
return
if __name__ == '__main__':
compare_bases_shape("TableTennis4DProDMP-v0", "TableTennis4DProMP-v0")
compare_bases_shape("fancy_ProDMP/TableTennis4D-v0", "fancy_ProMP/TableTennis4D-v0")
fancy_gym/examples/pd_control_gain_tuning.py
+5 -4
View File
@@ -3,19 +3,20 @@ from collections import OrderedDict
import numpy as np
from matplotlib import pyplot as plt
import gymnasium as gym
import fancy_gym
# This might work for some environments, however, please verify either way the correct trajectory information
# for your environment are extracted below
SEED = 1
env_id = "Reacher5dProMP-v0"
env_id = "fancy_ProMP/Reacher5d-v0"
env = fancy_gym.make(env_id, seed=SEED, controller_kwargs={'p_gains': 0.05, 'd_gains': 0.05}).env
env = fancy_gym.make(env_id, mp_config_override={'controller_kwargs': {'p_gains': 0.05, 'd_gains': 0.05}}).env
env.action_space.seed(SEED)
# Plot difference between real trajectory and target MP trajectory
env.reset()
env.reset(seed=SEED)
w = env.action_space.sample()
pos, vel = env.get_trajectory(w)
@@ -34,7 +35,7 @@ fig.show()
for t, (des_pos, des_vel) in enumerate(zip(pos, vel)):
actions = env.tracking_controller.get_action(des_pos, des_vel, env.current_pos, env.current_vel)
actions = np.clip(actions, env.env.action_space.low, env.env.action_space.high)
_, _, _, _ = env.env.step(actions)
env.env.step(actions)
if t % 15 == 0:
img.set_data(env.env.render(mode="rgb_array"))
fig.canvas.draw()