dodox/fancy_gym
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

unified API wrapper and updated examples

Browse Source
This commit is contained in:
ottofabian
2021-07-02 13:09:56 +02:00
parent 6607d9cff9
commit 80933eba09
22 changed files with 383 additions and 485 deletions
Download Patch File Download Diff File Expand all files Collapse all files
alr_envs/examples/examples_dmc.py
+46 -20
View File
@@ -1,9 +1,23 @@
from alr_envs.dmc.Ball_in_the_cup_mp_wrapper import DMCBallInCupMPWrapper
from alr_envs.dmc.ball_in_cup.ball_in_the_cup_mp_wrapper import DMCBallInCupMPWrapper
from alr_envs.utils.make_env_helpers import make_dmp_env, make_env
def example_dmc(env_name="fish-swim", seed=1, iterations=1000):
env = make_env(env_name, seed)
def example_dmc(env_id="fish-swim", seed=1, iterations=1000, render=True):
"""
Example for running a DMC based env in the step based setting.
The env_id has to be specified as `domain_name-task_name` or
for manipulation tasks as `manipulation-environment_name`
Args:
env_id: Either `domain_name-task_name` or `manipulation-environment_name`
seed: seed for deterministic behaviour
iterations: Number of rollout steps to run
render: Render the episode
Returns:
"""
env = make_env(env_id, seed)
rewards = 0
obs = env.reset()
print("observation shape:", env.observation_space.shape)
@@ -15,39 +29,44 @@ def example_dmc(env_name="fish-swim", seed=1, iterations=1000):
obs, reward, done, info = env.step(ac)
rewards += reward
env.render("human")
if render:
env.render("human")
if done:
print(env_name, rewards)
print(env_id, rewards)
rewards = 0
obs = env.reset()
env.close()
def example_custom_dmc_and_mp(seed=1):
def example_custom_dmc_and_mp(seed=1, iterations=1, render=True):
"""
Example for running a custom motion primitive based environments based off of a dmc task.
Our already registered environments follow the same structure, but do not directly allow for modifications.
Hence, this also allows to adjust hyperparameters of the motion primitives more easily.
Example for running a custom motion primitive based environments.
Our already registered environments follow the same structure.
Hence, this also allows to adjust hyperparameters of the motion primitives.
Yet, we recommend the method above if you are just interested in chaining those parameters for existing tasks.
We appreciate PRs for custom environments (especially MP wrappers of existing tasks)
for our repo: https://github.com/ALRhub/alr_envs/
Args:
seed: seed
seed: seed for deterministic behaviour
iterations: Number of rollout steps to run
render: Render the episode
Returns:
"""
# Base DMC name, according to structure of above example
base_env = "ball_in_cup-catch"
# Replace this wrapper with the custom wrapper for your environment by inheriting from the MPEnvWrapper.
# You can also add other gym.Wrappers in case they are needed.
# wrappers = [HoleReacherMPWrapper]
wrappers = [DMCBallInCupMPWrapper]
mp_kwargs = {
"num_dof": 2, # env.start_pos
"num_basis": 5,
"duration": 2,
"duration": 20,
"learn_goal": True,
"alpha_phase": 2,
"bandwidth_factor": 2,
@@ -57,14 +76,21 @@ def example_custom_dmc_and_mp(seed=1):
}
env = make_dmp_env(base_env, wrappers=wrappers, seed=seed, mp_kwargs=mp_kwargs)
# OR for a deterministic ProMP:
# env = make_detpmp_env(base_env, wrappers=wrappers, seed=seed, **mp_args)
# env = make_detpmp_env(base_env, wrappers=wrappers, seed=seed, mp_kwargs=mp_args)
# This renders the full MP trajectory
# It is only required to call render() once in the beginning, which renders every consecutive trajectory.
# Resetting to no rendering, can be achieved by render(mode=None).
# It is also possible to change them mode multiple times when
# e.g. only every nth trajectory should be displayed.
if render:
env.render(mode="human")
rewards = 0
obs = env.reset()
env.render("human")
# number of samples/full trajectories (multiple environment steps)
for i in range(10):
for i in range(iterations):
ac = env.action_space.sample()
obs, reward, done, info = env.step(ac)
rewards += reward
@@ -85,14 +111,14 @@ if __name__ == '__main__':
# export MUJOCO_GL="osmesa"
# Standard DMC Suite tasks
example_dmc("fish-swim", seed=10, iterations=100)
example_dmc("fish-swim", seed=10, iterations=1000, render=True)
# Manipulation tasks
# The vision versions are currently not integrated
example_dmc("manipulation-reach_site_features", seed=10, iterations=100)
# Disclaimer: The vision versions are currently not integrated and yield an error
example_dmc("manipulation-reach_site_features", seed=10, iterations=250, render=True)
# Gym + DMC hybrid task provided in the MP framework
example_dmc("dmc_ball_in_cup_dmp-v0", seed=10, iterations=10)
example_dmc("dmc_ball_in_cup-catch_detpmp-v0", seed=10, iterations=1, render=True)
# Custom DMC task
example_custom_dmc_and_mp()
example_custom_dmc_and_mp(seed=10, iterations=1, render=True)
alr_envs/examples/examples_general.py
+61 -28
View File
@@ -4,14 +4,23 @@ from collections import defaultdict
import gym
import numpy as np
from alr_envs.utils.make_env_helpers import make_env
from alr_envs.utils.make_env_helpers import make_env, make_env_rank
from alr_envs.utils.mp_env_async_sampler import AlrContextualMpEnvSampler, AlrMpEnvSampler, DummyDist
def example_general(env_id: str, seed=1, iterations=1000):
def example_general(env_id="Pendulum-v0", seed=1, iterations=1000, render=True):
"""
Example for running any env in the step based setting.
This also includes DMC environments when leveraging our custom make_env function.
Args:
env_id: OpenAI/Custom gym task id or either `domain_name-task_name` or `manipulation-environment_name` for DMC tasks
seed: seed for deterministic behaviour
iterations: Number of rollout steps to run
render: Render the episode
Returns:
"""
env = make_env(env_id, seed)
@@ -25,7 +34,7 @@ def example_general(env_id: str, seed=1, iterations=1000):
obs, reward, done, info = env.step(env.action_space.sample())
rewards += reward
if i % 1 == 0:
if render:
env.render()
if done:
@@ -34,36 +43,60 @@ def example_general(env_id: str, seed=1, iterations=1000):
obs = env.reset()
def example_async(env_id="alr_envs:HoleReacherDMP-v0", n_cpu=4, seed=int('533D', 16)):
def sample(env: gym.vector.VectorEnv, n_samples=100):
# for plotting
rewards = np.zeros(n_cpu)
def example_async(env_id="alr_envs:HoleReacher-v0", n_cpu=4, seed=int('533D', 16), n_samples=800):
"""
Example for running any env in a vectorized multiprocessing setting to generate more samples faster.
This also includes DMC and DMP environments when leveraging our custom make_env function.
Be aware, increasing the number of environments reduces the total length of the individual episodes.
# this would generate more samples than requested if n_samples % num_envs != 0
repeat = int(np.ceil(n_samples / env.num_envs))
vals = defaultdict(list)
for i in range(repeat):
obs, reward, done, info = envs.step(envs.action_space.sample())
vals['obs'].append(obs)
vals['reward'].append(reward)
vals['done'].append(done)
vals['info'].append(info)
rewards += reward
if np.any(done):
print(rewards[done])
rewards[done] = 0
Args:
env_id: OpenAI/Custom gym task id or either `domain_name-task_name` or `manipulation-environment_name` for DMC tasks
seed: seed for deterministic behaviour
n_cpu: Number of cpus cores to use in parallel
n_samples: number of samples generated in total by all environments.
# do not return values above threshold
return (*map(lambda v: np.stack(v)[:n_samples], vals.values()),)
Returns: Tuple of (obs, reward, done, info) with type np.ndarray
from alr_envs.utils.make_env_helpers import make_env_rank
envs = gym.vector.AsyncVectorEnv([make_env_rank(env_id, seed, i) for i in range(n_cpu)])
"""
env = gym.vector.AsyncVectorEnv([make_env_rank(env_id, seed, i) for i in range(n_cpu)])
# OR
# envs = gym.vector.AsyncVectorEnv([make_env(env_id, seed + i) for i in range(n_cpu)])
obs = envs.reset()
print(sample(envs, 16))
# for plotting
rewards = np.zeros(n_cpu)
buffer = defaultdict(list)
obs = env.reset()
# this would generate more samples than requested if n_samples % num_envs != 0
repeat = int(np.ceil(n_samples / env.num_envs))
for i in range(repeat):
obs, reward, done, info = env.step(env.action_space.sample())
buffer['obs'].append(obs)
buffer['reward'].append(reward)
buffer['done'].append(done)
buffer['info'].append(info)
rewards += reward
if np.any(done):
print(f"Reward at iteration {i}: {rewards[done]}")
rewards[done] = 0
# do not return values above threshold
return *map(lambda v: np.stack(v)[:n_samples], buffer.values()),
if __name__ == '__main__':
# Mujoco task from framework
example_general("alr_envs:ALRReacher-v0")
# Basic gym task
# example_general("Pendulum-v0", seed=10, iterations=200, render=True)
#
# # Basis task from framework
# example_general("alr_envs:HoleReacher-v0", seed=10, iterations=200, render=True)
#
# # OpenAI Mujoco task
# example_general("HalfCheetah-v2", seed=10, render=True)
#
# # Mujoco task from framework
# example_general("alr_envs:ALRReacher-v0", seed=10, iterations=200, render=True)
# Vectorized multiprocessing environments
example_async(env_id="alr_envs:HoleReacher-v0", n_cpu=2, seed=int('533D', 16), n_samples=2 * 200)
alr_envs/examples/examples_motion_primitives.py
+79 -37
View File
@@ -2,12 +2,14 @@ from alr_envs import HoleReacherMPWrapper
from alr_envs.utils.make_env_helpers import make_dmp_env, make_env
def example_mp(env_name="alr_envs:HoleReacherDMP-v1", seed=1):
def example_mp(env_name="alr_envs:HoleReacherDMP-v1", seed=1, iterations=1, render=True):
"""
Example for running a motion primitive based environment, which is already registered
Args:
env_name: DMP env_id
seed: seed
seed: seed for deterministic behaviour
iterations: Number of rollout steps to run
render: Render the episode
Returns:
@@ -16,44 +18,81 @@ def example_mp(env_name="alr_envs:HoleReacherDMP-v1", seed=1):
# First, it already takes care of seeding and second enables the use of DMC tasks within the gym interface.
env = make_env(env_name, seed)
# Changing the mp_kwargs is possible by providing them to gym.
# E.g. here by providing way to many basis functions
# mp_kwargs = {
# "num_dof": 5,
# "num_basis": 1000,
# "duration": 2,
# "learn_goal": True,
# "alpha_phase": 2,
# "bandwidth_factor": 2,
# "policy_type": "velocity",
# "weights_scale": 50,
# "goal_scale": 0.1
# }
# env = make_env(env_name, seed, mp_kwargs=mp_kwargs)
rewards = 0
# env.render(mode=None)
obs = env.reset()
# number of samples/full trajectories (multiple environment steps)
for i in range(10):
for i in range(iterations):
if render and i % 2 == 0:
# This renders the full MP trajectory
# It is only required to call render() once in the beginning, which renders every consecutive trajectory.
# Resetting to no rendering, can be achieved by render(mode=None).
# It is also possible to change the mode multiple times when
# e.g. only every second trajectory should be displayed, such as here
# Just make sure the correct mode is set before executing the step.
env.render(mode="human")
else:
env.render(mode=None)
ac = env.action_space.sample()
obs, reward, done, info = env.step(ac)
rewards += reward
if i % 1 == 0:
# render full DMP trajectory
# render can only be called once in the beginning as well. That would render every trajectory
# Calling it after every trajectory allows to modify the mode. mode=None, disables rendering.
env.render(mode="human")
if done:
print(rewards)
rewards = 0
obs = env.reset()
def example_custom_mp(seed=1):
def example_custom_mp(env_name="alr_envs:HoleReacherDMP-v1", seed=1, iterations=1, render=True):
"""
Example for running a motion primitive based environment, which is already registered
Args:
env_name: DMP env_id
seed: seed for deterministic behaviour
iterations: Number of rollout steps to run
render: Render the episode
Returns:
"""
# Changing the mp_kwargs is possible by providing them to gym.
# E.g. here by providing way to many basis functions
mp_kwargs = {
"num_dof": 5,
"num_basis": 1000,
"duration": 2,
"learn_goal": True,
"alpha_phase": 2,
"bandwidth_factor": 2,
"policy_type": "velocity",
"weights_scale": 50,
"goal_scale": 0.1
}
env = make_env(env_name, seed, mp_kwargs=mp_kwargs)
# This time rendering every trajectory
if render:
env.render(mode="human")
rewards = 0
obs = env.reset()
# number of samples/full trajectories (multiple environment steps)
for i in range(iterations):
ac = env.action_space.sample()
obs, reward, done, info = env.step(ac)
rewards += reward
if done:
print(rewards)
rewards = 0
obs = env.reset()
def example_fully_custom_mp(seed=1, iterations=1, render=True):
"""
Example for running a custom motion primitive based environments.
Our already registered environments follow the same structure.
@@ -63,12 +102,15 @@ def example_custom_mp(seed=1):
for our repo: https://github.com/ALRhub/alr_envs/
Args:
seed: seed
iterations: Number of rollout steps to run
render: Render the episode
Returns:
"""
base_env = "alr_envs:HoleReacher-v1"
# Replace this wrapper with the custom wrapper for your environment by inheriting from the MPEnvWrapper.
# You can also add other gym.Wrappers in case they are needed.
wrappers = [HoleReacherMPWrapper]
@@ -85,19 +127,16 @@ def example_custom_mp(seed=1):
}
env = make_dmp_env(base_env, wrappers=wrappers, seed=seed, mp_kwargs=mp_kwargs)
# OR for a deterministic ProMP:
# env = make_detpmp_env(base_env, wrappers=wrappers, seed=seed)
# env = make_detpmp_env(base_env, wrappers=wrappers, seed=seed, mp_kwargs=mp_kwargs)
if render:
env.render(mode="human")
rewards = 0
# render full DMP trajectory
# It is only required to call render() once in the beginning, which renders every consecutive trajectory.
# Resetting to no rendering, can be achieved by render(mode=None).
# It is also possible to change them mode multiple times when
# e.g. only every nth trajectory should be displayed.
env.render(mode="human")
obs = env.reset()
# number of samples/full trajectories (multiple environment steps)
for i in range(10):
for i in range(iterations):
ac = env.action_space.sample()
obs, reward, done, info = env.step(ac)
rewards += reward
@@ -110,10 +149,13 @@ def example_custom_mp(seed=1):
if __name__ == '__main__':
# DMP
example_mp("alr_envs:HoleReacherDMP-v1")
example_mp("alr_envs:HoleReacherDMP-v1", seed=10, iterations=1, render=True)
# DetProMP
example_mp("alr_envs:HoleReacherDetPMP-v1")
example_mp("alr_envs:HoleReacherDetPMP-v1", seed=10, iterations=1, render=True)
# Custom DMP
example_custom_mp()
# Altered basis functions
example_custom_mp("alr_envs:HoleReacherDMP-v1", seed=10, iterations=1, render=True)
# Custom MP
example_fully_custom_mp(seed=10, iterations=1, render=True)