From 9651d9e21352a191b3d01ebf9d009667a7b5e13c Mon Sep 17 00:00:00 2001 From: Fabian Date: Mon, 26 Sep 2022 08:39:54 +0200 Subject: [PATCH] updated test to pytest --- test/test_custom_envs.py | 35 ++++++++++++++++ test/test_dmc_envs.py | 61 +++++++++++++++------------ test/test_envs.py | 91 ---------------------------------------- test/utils.py | 56 ++++++++++++------------- 4 files changed, 96 insertions(+), 147 deletions(-) create mode 100644 test/test_custom_envs.py delete mode 100644 test/test_envs.py diff --git a/test/test_custom_envs.py b/test/test_custom_envs.py new file mode 100644 index 0000000..122f8a5 --- /dev/null +++ b/test/test_custom_envs.py @@ -0,0 +1,35 @@ +import fancy_gym +import gym +import pytest + +from test.utils import run_env, run_env_determinism + +CUSTOM_IDS = [spec.id for spec in gym.envs.registry.all() if + "fancy_gym" in spec.entry_point and 'make_bb_env_helper' not in spec.entry_point] +SEED = 1 + + +@pytest.mark.parametrize('env_id', CUSTOM_IDS) +def test_step_functionality(env_id: str): + """Tests that step environments run without errors using random actions.""" + run_env(env_id) + + +@pytest.mark.parametrize('env_id', CUSTOM_IDS) +def test_step_determinism(env_id: str): + """Tests that for step environments identical seeds produce identical trajectories.""" + seed = 0 + run_env_determinism(env_id, seed) + + +@pytest.mark.parametrize('env_id', fancy_gym.ALL_FANCY_MOVEMENT_PRIMITIVE_ENVIRONMENTS.values()) +def test_bb_functionality(env_id: str): + """Tests that black box environments run without errors using random actions.""" + run_env(env_id) + + +@pytest.mark.parametrize('env_id', fancy_gym.ALL_FANCY_MOVEMENT_PRIMITIVE_ENVIRONMENTS.values()) +def test_bb_determinism(env_id: str): + """Tests that for black box environment identical seeds produce identical trajectories.""" + seed = 0 + run_env_determinism(env_id, seed) diff --git a/test/test_dmc_envs.py b/test/test_dmc_envs.py index 000b31e..7293030 100644 --- a/test/test_dmc_envs.py +++ b/test/test_dmc_envs.py @@ -1,42 +1,49 @@ -import numpy as np +from typing import Tuple +import fancy_gym import pytest - from dm_control import suite, manipulation -DMC_ENVS = [f'{env}-{task}' for env, task in suite.ALL_TASKS if env != "lqr"] -MANIPULATION_SPECS = [f'manipulation-{task}' for task in manipulation.ALL if task.endswith('_features')] +from test.utils import run_env_determinism, run_env + +SUITE_IDS = [f'dmc:{env}-{task}' for env, task in suite.ALL_TASKS if env != "lqr"] +MANIPULATION_IDS = [f'dmc:manipulation-{task}' for task in manipulation.ALL if task.endswith('_features')] SEED = 1 -@pytest.mark.parametrize('env_id', DMC_ENVS) -def test_dmc_functionality(self, env_id: str): - """Tests that environments runs without errors using random actions.""" - self.run_env(env_id) +@pytest.mark.parametrize('env_id', SUITE_IDS) +def test_step_suite_functionality(env_id: str): + """Tests that suite step environments run without errors using random actions.""" + run_env(env_id) -@pytest.mark.parametrize('env_id', DMC_ENVS) -def test_dmc_determinism(self, env_id: str): - """Tests that identical seeds produce identical trajectories.""" +@pytest.mark.parametrize('env_id', SUITE_IDS) +def test_step_suite_determinism(env_id: str): + """Tests that for step environments identical seeds produce identical trajectories.""" seed = 0 - self._run_env_determinism(env_id, seed) + run_env_determinism(env_id, seed) -@pytest.mark.parametrize('env_id', MANIPULATION_SPECS) -def test_manipulation_functionality(self, env_id: str): - """Tests that environments runs without errors using random actions.""" - self.run_env(env_id) +@pytest.mark.parametrize('env_id', MANIPULATION_IDS) +def test_step_manipulation_functionality(env_id: str): + """Tests that manipulation step environments run without errors using random actions.""" + run_env(env_id) -@pytest.mark.parametrize('env_id', MANIPULATION_SPECS) -def test_manipulation_determinism(self, env_id: str): - """Tests that identical seeds produce identical trajectories.""" +@pytest.mark.parametrize('env_id', MANIPULATION_IDS) +def test_step_manipulation_determinism(env_id: str): + """Tests that for step environments identical seeds produce identical trajectories.""" seed = 0 - # Iterate over two trajectories, which should have the same state and action sequence - traj1 = self.run_env(env_id, seed=seed) - traj2 = self.run_env(env_id, seed=seed) - for i, time_step in enumerate(zip(*traj1, *traj2)): - obs1, rwd1, done1, obs2, rwd2, done2 = time_step - assert np.array_equal(obs1, obs2), f"Observations [{i}] {obs1} and {obs2} do not match." - assert np.all(rwd1 == rwd2), f"Rewards [{i}] {rwd1} and {rwd2} do not match." - assert np.all(done1 == done2), f"Dones [{i}] {done1} and {done2} do not match." + run_env_determinism(env_id, seed) + + +@pytest.mark.parametrize('env_id', [fancy_gym.ALL_DMC_MOVEMENT_PRIMITIVE_ENVIRONMENTS.values()]) +def test_bb_dmc_functionality(env_id: str): + """Tests that black box environments run without errors using random actions.""" + run_env(env_id) + + +@pytest.mark.parametrize('env_id', [fancy_gym.ALL_DMC_MOVEMENT_PRIMITIVE_ENVIRONMENTS.values()]) +def test_bb_dmc_determinism(env_id: str): + """Tests that for black box environment identical seeds produce identical trajectories.""" + run_env_determinism(env_id) diff --git a/test/test_envs.py b/test/test_envs.py deleted file mode 100644 index e8ae429..0000000 --- a/test/test_envs.py +++ /dev/null @@ -1,91 +0,0 @@ -import fancy_gym -import gym -import pytest - -from test.utils import run_env - -ALL_SPECS = list(spec for spec in gym.envs.registry.all() if "alr_envs" in spec.entry_point) -SEED = 1 - - -@pytest.mark.parametrize('env_id', fancy_gym.ALL_FANCY_MOVEMENT_PRIMITIVE_ENVIRONMENTS['DMP']) -def test_custom_dmp_functionality(env_id): - """Tests that environments runs without errors using random actions for custom DMP envs.""" - run_env(env_id) - - -@pytest.mark.parametrize('env_id', fancy_gym.ALL_FANCY_MOVEMENT_PRIMITIVE_ENVIRONMENTS['ProMP']) -def test_custom_promp_functionality(env_id): - """Tests that environments runs without errors using random actions for custom ProMP envs.""" - run_env(env_id) - - -def test_openai_environment_functionality(self): - """Tests that environments runs without errors using random actions for OpenAI gym MP envs.""" - with self.subTest(msg="DMP"): - for env_id in alr_envs.ALL_GYM_MOTION_PRIMITIVE_ENVIRONMENTS['DMP']: - with self.subTest(msg=env_id): - self.run_env(env_id) - - with self.subTest(msg="ProMP"): - for env_id in alr_envs.ALL_GYM_MOTION_PRIMITIVE_ENVIRONMENTS['ProMP']: - with self.subTest(msg=env_id): - self.run_env(env_id) - - -def test_dmc_environment_functionality(self): - """Tests that environments runs without errors using random actions for DMC MP envs.""" - with self.subTest(msg="DMP"): - for env_id in alr_envs.ALL_DEEPMIND_MOTION_PRIMITIVE_ENVIRONMENTS['DMP']: - with self.subTest(msg=env_id): - self.run_env(env_id) - - with self.subTest(msg="ProMP"): - for env_id in alr_envs.ALL_DEEPMIND_MOTION_PRIMITIVE_ENVIRONMENTS['ProMP']: - with self.subTest(msg=env_id): - self.run_env(env_id) - - -def test_metaworld_environment_functionality(self): - """Tests that environments runs without errors using random actions for Metaworld MP envs.""" - with self.subTest(msg="DMP"): - for env_id in alr_envs.ALL_METAWORLD_MOTION_PRIMITIVE_ENVIRONMENTS['DMP']: - with self.subTest(msg=env_id): - self.run_env(env_id) - - with self.subTest(msg="ProMP"): - for env_id in alr_envs.ALL_METAWORLD_MOTION_PRIMITIVE_ENVIRONMENTS['ProMP']: - with self.subTest(msg=env_id): - self.run_env(env_id) - - -def test_alr_environment_determinism(self): - """Tests that identical seeds produce identical trajectories for ALR MP Envs.""" - with self.subTest(msg="DMP"): - self._run_env_determinism(alr_envs.ALL_ALR_MOTION_PRIMITIVE_ENVIRONMENTS["DMP"]) - with self.subTest(msg="ProMP"): - self._run_env_determinism(alr_envs.ALL_ALR_MOTION_PRIMITIVE_ENVIRONMENTS["ProMP"]) - - -def test_openai_environment_determinism(self): - """Tests that identical seeds produce identical trajectories for OpenAI gym MP Envs.""" - with self.subTest(msg="DMP"): - self._run_env_determinism(alr_envs.ALL_GYM_MOTION_PRIMITIVE_ENVIRONMENTS["DMP"]) - with self.subTest(msg="ProMP"): - self._run_env_determinism(alr_envs.ALL_GYM_MOTION_PRIMITIVE_ENVIRONMENTS["ProMP"]) - - -def test_dmc_environment_determinism(self): - """Tests that identical seeds produce identical trajectories for DMC MP Envs.""" - with self.subTest(msg="DMP"): - self._run_env_determinism(alr_envs.ALL_DEEPMIND_MOTION_PRIMITIVE_ENVIRONMENTS["DMP"]) - with self.subTest(msg="ProMP"): - self._run_env_determinism(alr_envs.ALL_DEEPMIND_MOTION_PRIMITIVE_ENVIRONMENTS["ProMP"]) - - -def test_metaworld_environment_determinism(self): - """Tests that identical seeds produce identical trajectories for Metaworld MP Envs.""" - with self.subTest(msg="DMP"): - self._run_env_determinism(alr_envs.ALL_METAWORLD_MOTION_PRIMITIVE_ENVIRONMENTS["DMP"]) - with self.subTest(msg="ProMP"): - self._run_env_determinism(alr_envs.ALL_METAWORLD_MOTION_PRIMITIVE_ENVIRONMENTS["ProMP"]) diff --git a/test/utils.py b/test/utils.py index a558044..6e57e8a 100644 --- a/test/utils.py +++ b/test/utils.py @@ -3,46 +3,43 @@ import numpy as np from fancy_gym import make -def run_env(env_id, iterations=None, seed=SEED, render=False): +def run_env(env_id, iterations=None, seed=0, render=False): """ Example for running a DMC based env in the step based setting. - The env_id has to be specified as `domain_name-task_name` or + The env_id has to be specified as `dmc:domain_name-task_name` or for manipulation tasks as `manipulation-environment_name` Args: - env_id: Either `domain_name-task_name` or `manipulation-environment_name` + env_id: Either `dmc:domain_name-task_name` or `dmc:manipulation-environment_name` iterations: Number of rollout steps to run - seed= random seeding + seed: random seeding render: Render the episode - Returns: + Returns: observations, rewards, dones, actions """ env: gym.Env = make(env_id, seed=seed) rewards = [] observations = [] + actions = [] dones = [] obs = env.reset() - _verify_observations(obs, env.observation_space, "reset()") + verify_observations(obs, env.observation_space, "reset()") - length = env.spec.max_episode_steps - if iterations is None: - if length is None: - iterations = 1 - else: - iterations = length + iterations = iterations or (env.spec.max_episode_steps or 1) # number of samples(multiple environment steps) for i in range(iterations): observations.append(obs) ac = env.action_space.sample() + actions.append(ac) # ac = np.random.uniform(env.action_space.low, env.action_space.high, env.action_space.shape) obs, reward, done, info = env.step(ac) - _verify_observations(obs, env.observation_space, "step()") - _verify_reward(reward) - _verify_done(done) + verify_observations(obs, env.observation_space, "step()") + verify_reward(reward) + verify_done(done) rewards.append(reward) dones.append(done) @@ -51,34 +48,35 @@ def run_env(env_id, iterations=None, seed=SEED, render=False): env.render("human") if done: - obs = env.reset() + break - assert done, "Done flag is not True after max episode length." + assert done, "Done flag is not True after end of episode." observations.append(obs) env.close() del env - return np.array(observations), np.array(rewards), np.array(dones) + return np.array(observations), np.array(rewards), np.array(dones), np.array(actions) -def _run_env_determinism(self, env_id: str, seed: int): - traj1 = self.run_env(env_id, seed=seed) - traj2 = self.run_env(env_id, seed=seed) +def run_env_determinism(env_id: str, seed: int): + traj1 = run_env(env_id, seed=seed) + traj2 = run_env(env_id, seed=seed) # Iterate over two trajectories, which should have the same state and action sequence for i, time_step in enumerate(zip(*traj1, *traj2)): - obs1, rwd1, done1, obs2, rwd2, done2 = time_step - self.assertTrue(np.array_equal(obs1, obs2), f"Observations [{i}] {obs1} and {obs2} do not match.") - self.assertEqual(rwd1, rwd2, f"Rewards [{i}] {rwd1} and {rwd2} do not match.") - self.assertEqual(done1, done2, f"Dones [{i}] {done1} and {done2} do not match.") + obs1, rwd1, done1, ac1, obs2, rwd2, done2, ac2 = time_step + assert np.array_equal(obs1, obs2), f"Observations [{i}] {obs1} and {obs2} do not match." + assert np.array_equal(ac1, ac2), f"Actions [{i}] {ac1} and {ac2} do not match." + assert np.array_equal(rwd1, rwd2), f"Rewards [{i}] {rwd1} and {rwd2} do not match." + assert np.array_equal(done1, done2), f"Dones [{i}] {done1} and {done2} do not match." -def _verify_observations(obs, observation_space, obs_type="reset()"): +def verify_observations(obs, observation_space, obs_type="reset()"): assert observation_space.contains(obs), \ f"Observation {obs} received from {obs_type} not contained in observation space {observation_space}." -def _verify_reward(reward): - assert isinstance(reward, float), f"Returned {reward} as reward, expected float." +def verify_reward(reward): + assert isinstance(reward, (float, int)), f"Returned type {type(reward)} as reward, expected float or int." -def _verify_done(done): +def verify_done(done): assert isinstance(done, bool), f"Returned {done} as done flag, expected bool."