dodox/fancy_gym
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

updated test to pytest

Browse Source
This commit is contained in:
Fabian 2022-09-26 08:39:54 +02:00
parent 2875e07947
commit 9651d9e213
4 changed files with 96 additions and 147 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
test/test_custom_envs.py Normal file
View File

@ -0,0 +1,35 @@
import fancy_gym
import gym
import pytest
from test.utils import run_env, run_env_determinism
CUSTOM_IDS = [spec.id for spec in gym.envs.registry.all() if
"fancy_gym" in spec.entry_point and 'make_bb_env_helper' not in spec.entry_point]
SEED = 1
@pytest.mark.parametrize('env_id', CUSTOM_IDS)
def test_step_functionality(env_id: str):
"""Tests that step environments run without errors using random actions."""
run_env(env_id)
@pytest.mark.parametrize('env_id', CUSTOM_IDS)
def test_step_determinism(env_id: str):
"""Tests that for step environments identical seeds produce identical trajectories."""
seed = 0
run_env_determinism(env_id, seed)
@pytest.mark.parametrize('env_id', fancy_gym.ALL_FANCY_MOVEMENT_PRIMITIVE_ENVIRONMENTS.values())
def test_bb_functionality(env_id: str):
"""Tests that black box environments run without errors using random actions."""
run_env(env_id)
@pytest.mark.parametrize('env_id', fancy_gym.ALL_FANCY_MOVEMENT_PRIMITIVE_ENVIRONMENTS.values())
def test_bb_determinism(env_id: str):
"""Tests that for black box environment identical seeds produce identical trajectories."""
seed = 0
run_env_determinism(env_id, seed)

61
test/test_dmc_envs.py
View File

@ -1,42 +1,49 @@
import numpy as np from typing import Tuple
import fancy_gym
import pytest import pytest
from dm_control import suite, manipulation from dm_control import suite, manipulation
DMC_ENVS = [f'{env}-{task}' for env, task in suite.ALL_TASKS if env != "lqr"] from test.utils import run_env_determinism, run_env
MANIPULATION_SPECS = [f'manipulation-{task}' for task in manipulation.ALL if task.endswith('_features')]
SUITE_IDS = [f'dmc:{env}-{task}' for env, task in suite.ALL_TASKS if env != "lqr"]
MANIPULATION_IDS = [f'dmc:manipulation-{task}' for task in manipulation.ALL if task.endswith('_features')]
SEED = 1 SEED = 1
@pytest.mark.parametrize('env_id', DMC_ENVS) @pytest.mark.parametrize('env_id', SUITE_IDS)
def test_dmc_functionality(self, env_id: str): def test_step_suite_functionality(env_id: str):
"""Tests that environments runs without errors using random actions.""" """Tests that suite step environments run without errors using random actions."""
self.run_env(env_id) run_env(env_id)
@pytest.mark.parametrize('env_id', DMC_ENVS) @pytest.mark.parametrize('env_id', SUITE_IDS)
def test_dmc_determinism(self, env_id: str): def test_step_suite_determinism(env_id: str):
"""Tests that identical seeds produce identical trajectories.""" """Tests that for step environments identical seeds produce identical trajectories."""
seed = 0 seed = 0
self._run_env_determinism(env_id, seed) run_env_determinism(env_id, seed)
@pytest.mark.parametrize('env_id', MANIPULATION_SPECS) @pytest.mark.parametrize('env_id', MANIPULATION_IDS)
def test_manipulation_functionality(self, env_id: str): def test_step_manipulation_functionality(env_id: str):
"""Tests that environments runs without errors using random actions.""" """Tests that manipulation step environments run without errors using random actions."""
self.run_env(env_id) run_env(env_id)
@pytest.mark.parametrize('env_id', MANIPULATION_SPECS) @pytest.mark.parametrize('env_id', MANIPULATION_IDS)
def test_manipulation_determinism(self, env_id: str): def test_step_manipulation_determinism(env_id: str):
"""Tests that identical seeds produce identical trajectories.""" """Tests that for step environments identical seeds produce identical trajectories."""
seed = 0 seed = 0
# Iterate over two trajectories, which should have the same state and action sequence run_env_determinism(env_id, seed)
traj1 = self.run_env(env_id, seed=seed)
traj2 = self.run_env(env_id, seed=seed)
for i, time_step in enumerate(zip(*traj1, *traj2)): @pytest.mark.parametrize('env_id', [fancy_gym.ALL_DMC_MOVEMENT_PRIMITIVE_ENVIRONMENTS.values()])
obs1, rwd1, done1, obs2, rwd2, done2 = time_step def test_bb_dmc_functionality(env_id: str):
assert np.array_equal(obs1, obs2), f"Observations [{i}] {obs1} and {obs2} do not match." """Tests that black box environments run without errors using random actions."""
assert np.all(rwd1 == rwd2), f"Rewards [{i}] {rwd1} and {rwd2} do not match." run_env(env_id)
assert np.all(done1 == done2), f"Dones [{i}] {done1} and {done2} do not match."
@pytest.mark.parametrize('env_id', [fancy_gym.ALL_DMC_MOVEMENT_PRIMITIVE_ENVIRONMENTS.values()])
def test_bb_dmc_determinism(env_id: str):
"""Tests that for black box environment identical seeds produce identical trajectories."""
run_env_determinism(env_id)

91
test/test_envs.py
View File

@ -1,91 +0,0 @@
import fancy_gym
import gym
import pytest
from test.utils import run_env
ALL_SPECS = list(spec for spec in gym.envs.registry.all() if "alr_envs" in spec.entry_point)
SEED = 1
@pytest.mark.parametrize('env_id', fancy_gym.ALL_FANCY_MOVEMENT_PRIMITIVE_ENVIRONMENTS['DMP'])
def test_custom_dmp_functionality(env_id):
"""Tests that environments runs without errors using random actions for custom DMP envs."""
run_env(env_id)
@pytest.mark.parametrize('env_id', fancy_gym.ALL_FANCY_MOVEMENT_PRIMITIVE_ENVIRONMENTS['ProMP'])
def test_custom_promp_functionality(env_id):
"""Tests that environments runs without errors using random actions for custom ProMP envs."""
run_env(env_id)
def test_openai_environment_functionality(self):
"""Tests that environments runs without errors using random actions for OpenAI gym MP envs."""
with self.subTest(msg="DMP"):
for env_id in alr_envs.ALL_GYM_MOTION_PRIMITIVE_ENVIRONMENTS['DMP']:
with self.subTest(msg=env_id):
self.run_env(env_id)
with self.subTest(msg="ProMP"):
for env_id in alr_envs.ALL_GYM_MOTION_PRIMITIVE_ENVIRONMENTS['ProMP']:
with self.subTest(msg=env_id):
self.run_env(env_id)
def test_dmc_environment_functionality(self):
"""Tests that environments runs without errors using random actions for DMC MP envs."""
with self.subTest(msg="DMP"):
for env_id in alr_envs.ALL_DEEPMIND_MOTION_PRIMITIVE_ENVIRONMENTS['DMP']:
with self.subTest(msg=env_id):
self.run_env(env_id)
with self.subTest(msg="ProMP"):
for env_id in alr_envs.ALL_DEEPMIND_MOTION_PRIMITIVE_ENVIRONMENTS['ProMP']:
with self.subTest(msg=env_id):
self.run_env(env_id)
def test_metaworld_environment_functionality(self):
"""Tests that environments runs without errors using random actions for Metaworld MP envs."""
with self.subTest(msg="DMP"):
for env_id in alr_envs.ALL_METAWORLD_MOTION_PRIMITIVE_ENVIRONMENTS['DMP']:
with self.subTest(msg=env_id):
self.run_env(env_id)
with self.subTest(msg="ProMP"):
for env_id in alr_envs.ALL_METAWORLD_MOTION_PRIMITIVE_ENVIRONMENTS['ProMP']:
with self.subTest(msg=env_id):
self.run_env(env_id)
def test_alr_environment_determinism(self):
"""Tests that identical seeds produce identical trajectories for ALR MP Envs."""
with self.subTest(msg="DMP"):
self._run_env_determinism(alr_envs.ALL_ALR_MOTION_PRIMITIVE_ENVIRONMENTS["DMP"])
with self.subTest(msg="ProMP"):
self._run_env_determinism(alr_envs.ALL_ALR_MOTION_PRIMITIVE_ENVIRONMENTS["ProMP"])
def test_openai_environment_determinism(self):
"""Tests that identical seeds produce identical trajectories for OpenAI gym MP Envs."""
with self.subTest(msg="DMP"):
self._run_env_determinism(alr_envs.ALL_GYM_MOTION_PRIMITIVE_ENVIRONMENTS["DMP"])
with self.subTest(msg="ProMP"):
self._run_env_determinism(alr_envs.ALL_GYM_MOTION_PRIMITIVE_ENVIRONMENTS["ProMP"])
def test_dmc_environment_determinism(self):
"""Tests that identical seeds produce identical trajectories for DMC MP Envs."""
with self.subTest(msg="DMP"):
self._run_env_determinism(alr_envs.ALL_DEEPMIND_MOTION_PRIMITIVE_ENVIRONMENTS["DMP"])
with self.subTest(msg="ProMP"):
self._run_env_determinism(alr_envs.ALL_DEEPMIND_MOTION_PRIMITIVE_ENVIRONMENTS["ProMP"])
def test_metaworld_environment_determinism(self):
"""Tests that identical seeds produce identical trajectories for Metaworld MP Envs."""
with self.subTest(msg="DMP"):
self._run_env_determinism(alr_envs.ALL_METAWORLD_MOTION_PRIMITIVE_ENVIRONMENTS["DMP"])
with self.subTest(msg="ProMP"):
self._run_env_determinism(alr_envs.ALL_METAWORLD_MOTION_PRIMITIVE_ENVIRONMENTS["ProMP"])

56
test/utils.py
View File

@ -3,46 +3,43 @@ import numpy as np
from fancy_gym import make from fancy_gym import make
def run_env(env_id, iterations=None, seed=SEED, render=False): def run_env(env_id, iterations=None, seed=0, render=False):
""" """
Example for running a DMC based env in the step based setting. Example for running a DMC based env in the step based setting.
The env_id has to be specified as `domain_name-task_name` or The env_id has to be specified as `dmc:domain_name-task_name` or
for manipulation tasks as `manipulation-environment_name` for manipulation tasks as `manipulation-environment_name`
Args: Args:
env_id: Either `domain_name-task_name` or `manipulation-environment_name` env_id: Either `dmc:domain_name-task_name` or `dmc:manipulation-environment_name`
iterations: Number of rollout steps to run iterations: Number of rollout steps to run
seed= random seeding seed: random seeding
render: Render the episode render: Render the episode
Returns: Returns: observations, rewards, dones, actions
""" """
env: gym.Env = make(env_id, seed=seed) env: gym.Env = make(env_id, seed=seed)
rewards = [] rewards = []
observations = [] observations = []
actions = []
dones = [] dones = []
obs = env.reset() obs = env.reset()
_verify_observations(obs, env.observation_space, "reset()") verify_observations(obs, env.observation_space, "reset()")
length = env.spec.max_episode_steps iterations = iterations or (env.spec.max_episode_steps or 1)
if iterations is None:
if length is None:
iterations = 1
else:
iterations = length
# number of samples(multiple environment steps) # number of samples(multiple environment steps)
for i in range(iterations): for i in range(iterations):
observations.append(obs) observations.append(obs)
ac = env.action_space.sample() ac = env.action_space.sample()
actions.append(ac)
# ac = np.random.uniform(env.action_space.low, env.action_space.high, env.action_space.shape) # ac = np.random.uniform(env.action_space.low, env.action_space.high, env.action_space.shape)
obs, reward, done, info = env.step(ac) obs, reward, done, info = env.step(ac)
_verify_observations(obs, env.observation_space, "step()") verify_observations(obs, env.observation_space, "step()")
_verify_reward(reward) verify_reward(reward)
_verify_done(done) verify_done(done)
rewards.append(reward) rewards.append(reward)
dones.append(done) dones.append(done)
@ -51,34 +48,35 @@ def run_env(env_id, iterations=None, seed=SEED, render=False):
env.render("human") env.render("human")
if done: if done:
obs = env.reset() break
assert done, "Done flag is not True after max episode length." assert done, "Done flag is not True after end of episode."
observations.append(obs) observations.append(obs)
env.close() env.close()
del env del env
return np.array(observations), np.array(rewards), np.array(dones) return np.array(observations), np.array(rewards), np.array(dones), np.array(actions)
def _run_env_determinism(self, env_id: str, seed: int): def run_env_determinism(env_id: str, seed: int):
traj1 = self.run_env(env_id, seed=seed) traj1 = run_env(env_id, seed=seed)
traj2 = self.run_env(env_id, seed=seed) traj2 = run_env(env_id, seed=seed)
# Iterate over two trajectories, which should have the same state and action sequence # Iterate over two trajectories, which should have the same state and action sequence
for i, time_step in enumerate(zip(*traj1, *traj2)): for i, time_step in enumerate(zip(*traj1, *traj2)):
obs1, rwd1, done1, obs2, rwd2, done2 = time_step obs1, rwd1, done1, ac1, obs2, rwd2, done2, ac2 = time_step
self.assertTrue(np.array_equal(obs1, obs2), f"Observations [{i}] {obs1} and {obs2} do not match.") assert np.array_equal(obs1, obs2), f"Observations [{i}] {obs1} and {obs2} do not match."
self.assertEqual(rwd1, rwd2, f"Rewards [{i}] {rwd1} and {rwd2} do not match.") assert np.array_equal(ac1, ac2), f"Actions [{i}] {ac1} and {ac2} do not match."
self.assertEqual(done1, done2, f"Dones [{i}] {done1} and {done2} do not match.") assert np.array_equal(rwd1, rwd2), f"Rewards [{i}] {rwd1} and {rwd2} do not match."
assert np.array_equal(done1, done2), f"Dones [{i}] {done1} and {done2} do not match."
def _verify_observations(obs, observation_space, obs_type="reset()"): def verify_observations(obs, observation_space, obs_type="reset()"):
assert observation_space.contains(obs), \ assert observation_space.contains(obs), \
f"Observation {obs} received from {obs_type} not contained in observation space {observation_space}." f"Observation {obs} received from {obs_type} not contained in observation space {observation_space}."
def _verify_reward(reward): def verify_reward(reward):
assert isinstance(reward, float), f"Returned {reward} as reward, expected float." assert isinstance(reward, (float, int)), f"Returned type {type(reward)} as reward, expected float or int."
def _verify_done(done): def verify_done(done):
assert isinstance(done, bool), f"Returned {done} as done flag, expected bool." assert isinstance(done, bool), f"Returned {done} as done flag, expected bool."