From 72fa14d786462e95c5f9b5ba653a16e714f65826 Mon Sep 17 00:00:00 2001
From: kngwyu <yuji.kngw.80s.revive@gmail.com>
Date: Sun, 11 Apr 2021 18:37:10 +0900
Subject: [PATCH 01/24] [Experimental] NoRewardRoom-v0

---
 mujoco_maze/maze_task.py | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)

diff --git a/mujoco_maze/maze_task.py b/mujoco_maze/maze_task.py
index e953cb1..828e140 100644
--- a/mujoco_maze/maze_task.py
+++ b/mujoco_maze/maze_task.py
@@ -329,6 +329,29 @@ class SubGoalTRoom(GoalRewardTRoom):
         )
 
 
+class NoRewardRoom(MazeTask):
+    REWARD_THRESHOLD: float = 0.0
+    MAZE_SIZE_SCALING: Scaling = Scaling(4.0, 4.0, 4.0)
+
+    def reward(self, obs: np.ndarray) -> float:
+        return 0.0
+
+    @staticmethod
+    def create_maze() -> List[List[MazeCell]]:
+        E, B, R = MazeCell.EMPTY, MazeCell.BLOCK, MazeCell.ROBOT
+        return [
+            [B, B, B, B, B, B, B, B, B],
+            [B, E, E, B, E, E, E, E, B],
+            [B, E, E, B, E, E, E, E, B],
+            [B, E, E, E, E, E, B, B, B],
+            [B, E, E, E, R, E, E, E, B],
+            [B, B, B, E, E, E, E, E, B],
+            [B, E, E, E, E, B, E, E, B],
+            [B, E, E, E, E, B, E, E, B],
+            [B, B, B, B, B, B, B, B, B],
+        ]
+
+
 class GoalRewardBlockMaze(GoalRewardUMaze):
     MAZE_SIZE_SCALING: Scaling = Scaling(8.0, 4.0, None)
     OBSERVE_BLOCKS: bool = True
@@ -460,6 +483,7 @@ class TaskRegistry:
         "4Rooms": [DistReward4Rooms, GoalReward4Rooms, SubGoal4Rooms],
         "TRoom": [DistRewardTRoom, GoalRewardTRoom, SubGoalTRoom],
         "BlockMaze": [DistRewardBlockMaze, GoalRewardBlockMaze],
+        "NoRewardRoom": [NoRewardRoom],
         "Billiard": [
             DistRewardBilliard,
             GoalRewardBilliard,

From 100432fceb6c894b2de2c44821d93b3275114c2a Mon Sep 17 00:00:00 2001
From: kngwyu <yuji.kngw.80s.revive@gmail.com>
Date: Mon, 12 Apr 2021 21:21:25 +0900
Subject: [PATCH 02/24] Alternative rendering

---
 mujoco_maze/alt_rendering.py | 92 ++++++++++++++++++++++++++++++++++++
 mujoco_maze/maze_env.py      | 60 ++++++++++++++++++-----
 mujoco_maze/point.py         |  2 +-
 3 files changed, 142 insertions(+), 12 deletions(-)
 create mode 100644 mujoco_maze/alt_rendering.py

diff --git a/mujoco_maze/alt_rendering.py b/mujoco_maze/alt_rendering.py
new file mode 100644
index 0000000..4aa65b9
--- /dev/null
+++ b/mujoco_maze/alt_rendering.py
@@ -0,0 +1,92 @@
+"""
+2D rendering framework
+"""
+import os
+import sys
+
+if "Apple" in sys.version:
+    if "DYLD_FALLBACK_LIBRARY_PATH" in os.environ:
+        os.environ["DYLD_FALLBACK_LIBRARY_PATH"] += ":/usr/lib"
+        # (JDS 2016/04/15): avoid bug on Anaconda 2.3.0 / Yosemite
+
+from gym import error
+
+
+import pyglet
+
+from pyglet import gl
+
+
+RAD2DEG = 57.29577951308232
+
+
+def get_display(spec):
+    """Convert a display specification (such as :0) into an actual Display
+    object.
+
+    Pyglet only supports multiple Displays on Linux.
+    """
+    if spec is None:
+        return pyglet.canvas.get_display()
+        # returns already available pyglet_display,
+        # if there is no pyglet display available then it creates one
+    elif isinstance(spec, str):
+        return pyglet.canvas.Display(spec)
+    else:
+        raise error.Error(
+            f"Invalid display specification: {spec}. (Must be a string like :0 or None.)"
+        )
+
+
+def get_window(width, height, display, **kwargs):
+    """
+    Will create a pyglet window from the display specification provided.
+    """
+    screen = display.get_screens()  # available screens
+    config = screen[0].get_best_config()  # selecting the first screen
+    context = config.create_context(None)  # create GL context
+
+    return pyglet.window.Window(
+        width=width,
+        height=height,
+        display=display,
+        config=config,
+        context=context,
+        **kwargs,
+    )
+
+
+class ImageViewer:
+    def __init__(self, width, height, display=None):
+        display = get_display(display)
+
+        self.width = width
+        self.height = height
+        self.window = get_window(width=width, height=height, display=display)
+        self.window.on_close = self.window_closed_by_user
+        self.isopen = True
+
+        gl.glEnable(gl.GL_BLEND)
+        gl.glBlendFunc(gl.GL_SRC_ALPHA, gl.GL_ONE_MINUS_SRC_ALPHA)
+
+    def imshow(self, arr):
+        assert len(arr.shape) == 3, "You passed in an image with the wrong number shape"
+        self.window.clear()
+        self.window.switch_to()
+        self.window.dispatch_events()
+        image = pyglet.image.ImageData(
+            arr.shape[1], arr.shape[0], "RGB", arr.tobytes(), pitch=arr.shape[1] * -3
+        )
+        image.blit(0, 0)  # draw
+        self.window.flip()
+
+    def close(self):
+        if self.isopen and sys.meta_path:
+            self.window.close()
+            self.isopen = False
+
+    def window_closed_by_user(self):
+        self.isopen = False
+
+    def __del__(self):
+        self.close()
diff --git a/mujoco_maze/maze_env.py b/mujoco_maze/maze_env.py
index 7619c35..51cc7d7 100644
--- a/mujoco_maze/maze_env.py
+++ b/mujoco_maze/maze_env.py
@@ -10,7 +10,7 @@ import itertools as it
 import os
 import tempfile
 import xml.etree.ElementTree as ET
-from typing import List, Tuple, Type
+from typing import Any, List, Tuple, Type
 
 import gym
 import numpy as np
@@ -33,7 +33,7 @@ class MazeEnv(gym.Env):
         inner_reward_scaling: float = 1.0,
         restitution_coef: float = 0.8,
         task_kwargs: dict = {},
-        *args,
+        use_alt_viewer: bool = False,
         **kwargs,
     ) -> None:
         self._task = maze_task(maze_size_scaling, **task_kwargs)
@@ -70,11 +70,19 @@ class MazeEnv(gym.Env):
             if model_cls.RADIUS is None:
                 raise ValueError("Manual collision needs radius of the model")
             self._collision = maze_env_utils.CollisionDetector(
-                structure, size_scaling, torso_x, torso_y, model_cls.RADIUS,
+                structure,
+                size_scaling,
+                torso_x,
+                torso_y,
+                model_cls.RADIUS,
             )
             # Now all object balls have size=1.0
             self._objball_collision = maze_env_utils.CollisionDetector(
-                structure, size_scaling, torso_x, torso_y, self._task.OBJECT_BALL_SIZE,
+                structure,
+                size_scaling,
+                torso_x,
+                torso_y,
+                self._task.OBJECT_BALL_SIZE,
             )
         else:
             self._collision = None
@@ -141,7 +149,15 @@ class MazeEnv(gym.Env):
                     # Movable block.
                     self.movable_blocks.append(f"movable_{i}_{j}")
                     _add_movable_block(
-                        worldbody, struct, i, j, size_scaling, x, y, h, height_offset,
+                        worldbody,
+                        struct,
+                        i,
+                        j,
+                        size_scaling,
+                        x,
+                        y,
+                        h,
+                        height_offset,
                     )
                 elif struct.is_object_ball():
                     # Movable Ball
@@ -173,8 +189,10 @@ class MazeEnv(gym.Env):
         _, file_path = tempfile.mkstemp(text=True, suffix=".xml")
         tree.write(file_path)
         self.world_tree = tree
-        self.wrapped_env = model_cls(*args, file_path=file_path, **kwargs)
+        self.wrapped_env = model_cls(file_path=file_path, **kwargs)
         self.observation_space = self._get_obs_space()
+        self._use_alt_viewer = use_alt_viewer
+        self._alt_viewer = None
 
     @property
     def has_extended_obs(self) -> bool:
@@ -337,11 +355,26 @@ class MazeEnv(gym.Env):
             self.data.site_xpos[idx][: len(goal.pos)] = goal.pos
 
     @property
-    def viewer(self):
-        return self.wrapped_env.viewer
+    def viewer(self) -> Any:
+        if self._use_alt_viewer:
+            return self._alt_viewer
+        else:
+            return self.wrapped_env.viewer
 
-    def render(self, *args, **kwargs):
-        return self.wrapped_env.render(*args, **kwargs)
+    def render(self, mode="human", **kwargs) -> Any:
+        if self._use_alt_viewer:
+            image = self.wrapped_env.sim.render(640, 480)[::-1, :, :]
+            if self._alt_viewer is None:
+                from mujoco_maze.alt_rendering import ImageViewer
+
+                self._alt_viewer = ImageViewer(640, 480)
+            if mode == "rgb_array":
+                return image
+            else:
+                self._alt_viewer.imshow(image)
+                return self._alt_viewer.isopen
+        else:
+            return self.wrapped_env.render(mode=mode, **kwargs)
 
     @property
     def action_space(self):
@@ -406,6 +439,8 @@ class MazeEnv(gym.Env):
 
     def close(self) -> None:
         self.wrapped_env.close()
+        if self._alt_viewer is not None:
+            self._alt_viewer.close()
 
 
 def _add_object_ball(
@@ -479,7 +514,10 @@ def _add_movable_block(
         shrink = 1.0
     size = size_scaling * 0.5 * shrink
     movable_body = ET.SubElement(
-        worldbody, "body", name=f"movable_{i}_{j}", pos=f"{x} {y} {h}",
+        worldbody,
+        "body",
+        name=f"movable_{i}_{j}",
+        pos=f"{x} {y} {h}",
     )
     ET.SubElement(
         movable_body,
diff --git a/mujoco_maze/point.py b/mujoco_maze/point.py
index 4e8c071..6f64603 100644
--- a/mujoco_maze/point.py
+++ b/mujoco_maze/point.py
@@ -22,7 +22,7 @@ class PointEnv(AgentModel):
 
     VELOCITY_LIMITS: float = 10.0
 
-    def __init__(self, file_path: Optional[str] = None):
+    def __init__(self, file_path: Optional[str] = None) -> None:
         super().__init__(file_path, 1)
         high = np.inf * np.ones(6, dtype=np.float32)
         high[3:] = self.VELOCITY_LIMITS * 1.2

From e12e38e64ea479228ab98c8533321c1d4a06226e Mon Sep 17 00:00:00 2001
From: kngwyu <yuji.kngw.80s.revive@gmail.com>
Date: Mon, 12 Apr 2021 22:11:53 +0900
Subject: [PATCH 03/24] Fix alternative rendering

---
 mujoco_maze/alt_rendering.py | 92 ------------------------------------
 mujoco_maze/maze_env.py      | 19 ++++++--
 2 files changed, 15 insertions(+), 96 deletions(-)
 delete mode 100644 mujoco_maze/alt_rendering.py

diff --git a/mujoco_maze/alt_rendering.py b/mujoco_maze/alt_rendering.py
deleted file mode 100644
index 4aa65b9..0000000
--- a/mujoco_maze/alt_rendering.py
+++ /dev/null
@@ -1,92 +0,0 @@
-"""
-2D rendering framework
-"""
-import os
-import sys
-
-if "Apple" in sys.version:
-    if "DYLD_FALLBACK_LIBRARY_PATH" in os.environ:
-        os.environ["DYLD_FALLBACK_LIBRARY_PATH"] += ":/usr/lib"
-        # (JDS 2016/04/15): avoid bug on Anaconda 2.3.0 / Yosemite
-
-from gym import error
-
-
-import pyglet
-
-from pyglet import gl
-
-
-RAD2DEG = 57.29577951308232
-
-
-def get_display(spec):
-    """Convert a display specification (such as :0) into an actual Display
-    object.
-
-    Pyglet only supports multiple Displays on Linux.
-    """
-    if spec is None:
-        return pyglet.canvas.get_display()
-        # returns already available pyglet_display,
-        # if there is no pyglet display available then it creates one
-    elif isinstance(spec, str):
-        return pyglet.canvas.Display(spec)
-    else:
-        raise error.Error(
-            f"Invalid display specification: {spec}. (Must be a string like :0 or None.)"
-        )
-
-
-def get_window(width, height, display, **kwargs):
-    """
-    Will create a pyglet window from the display specification provided.
-    """
-    screen = display.get_screens()  # available screens
-    config = screen[0].get_best_config()  # selecting the first screen
-    context = config.create_context(None)  # create GL context
-
-    return pyglet.window.Window(
-        width=width,
-        height=height,
-        display=display,
-        config=config,
-        context=context,
-        **kwargs,
-    )
-
-
-class ImageViewer:
-    def __init__(self, width, height, display=None):
-        display = get_display(display)
-
-        self.width = width
-        self.height = height
-        self.window = get_window(width=width, height=height, display=display)
-        self.window.on_close = self.window_closed_by_user
-        self.isopen = True
-
-        gl.glEnable(gl.GL_BLEND)
-        gl.glBlendFunc(gl.GL_SRC_ALPHA, gl.GL_ONE_MINUS_SRC_ALPHA)
-
-    def imshow(self, arr):
-        assert len(arr.shape) == 3, "You passed in an image with the wrong number shape"
-        self.window.clear()
-        self.window.switch_to()
-        self.window.dispatch_events()
-        image = pyglet.image.ImageData(
-            arr.shape[1], arr.shape[0], "RGB", arr.tobytes(), pitch=arr.shape[1] * -3
-        )
-        image.blit(0, 0)  # draw
-        self.window.flip()
-
-    def close(self):
-        if self.isopen and sys.meta_path:
-            self.window.close()
-            self.isopen = False
-
-    def window_closed_by_user(self):
-        self.isopen = False
-
-    def __del__(self):
-        self.close()
diff --git a/mujoco_maze/maze_env.py b/mujoco_maze/maze_env.py
index 51cc7d7..ac80885 100644
--- a/mujoco_maze/maze_env.py
+++ b/mujoco_maze/maze_env.py
@@ -363,15 +363,26 @@ class MazeEnv(gym.Env):
 
     def render(self, mode="human", **kwargs) -> Any:
         if self._use_alt_viewer:
-            image = self.wrapped_env.sim.render(640, 480)[::-1, :, :]
             if self._alt_viewer is None:
-                from mujoco_maze.alt_rendering import ImageViewer
+                from gym.envs.classic_control.rendering import SimpleImageViewer
+                import mujoco_py
 
-                self._alt_viewer = ImageViewer(640, 480)
+                self._mj_viewer = mujoco_py.MjRenderContextOffscreen(
+                    self.wrapped_env.sim
+                )
+                self._alt_viewer = SimpleImageViewer()
+
+            self._mj_viewer._set_mujoco_buffers()
+            self._mj_viewer.render(640, 480)
+            image = np.asarray(
+                self._mj_viewer.read_pixels(640, 480, depth=False)[::-1, :, :],
+                dtype=np.uint8,
+            )
             if mode == "rgb_array":
                 return image
             else:
-                self._alt_viewer.imshow(image)
+                if not (image.min() == 0 and image.max() == 0):
+                    self._alt_viewer.imshow(image)
                 return self._alt_viewer.isopen
         else:
             return self.wrapped_env.render(mode=mode, **kwargs)

From 9e16a68dc2b9a6d65209f1dcb6f84b129f1aae65 Mon Sep 17 00:00:00 2001
From: kngwyu <yuji.kngw.80s.revive@gmail.com>
Date: Tue, 13 Apr 2021 18:18:39 +0900
Subject: [PATCH 04/24] Websocket viewer

---
 mujoco_maze/maze_env.py | 100 ++++++++++++++++++++++++++++------------
 1 file changed, 70 insertions(+), 30 deletions(-)

diff --git a/mujoco_maze/maze_env.py b/mujoco_maze/maze_env.py
index ac80885..9f6d2e1 100644
--- a/mujoco_maze/maze_env.py
+++ b/mujoco_maze/maze_env.py
@@ -10,7 +10,7 @@ import itertools as it
 import os
 import tempfile
 import xml.etree.ElementTree as ET
-from typing import Any, List, Tuple, Type
+from typing import Any, List, Optional, Tuple, Type
 
 import gym
 import numpy as np
@@ -33,7 +33,7 @@ class MazeEnv(gym.Env):
         inner_reward_scaling: float = 1.0,
         restitution_coef: float = 0.8,
         task_kwargs: dict = {},
-        use_alt_viewer: bool = False,
+        websock_render_port: Optional[int] = None,
         **kwargs,
     ) -> None:
         self._task = maze_task(maze_size_scaling, **task_kwargs)
@@ -191,8 +191,9 @@ class MazeEnv(gym.Env):
         self.world_tree = tree
         self.wrapped_env = model_cls(file_path=file_path, **kwargs)
         self.observation_space = self._get_obs_space()
-        self._use_alt_viewer = use_alt_viewer
-        self._alt_viewer = None
+        self._websock_port = websock_port
+        self._mj_offscreen_viewer = None
+        self._websock_server_pipe = None
 
     @property
     def has_extended_obs(self) -> bool:
@@ -356,36 +357,75 @@ class MazeEnv(gym.Env):
 
     @property
     def viewer(self) -> Any:
-        if self._use_alt_viewer:
-            return self._alt_viewer
+        if self._websock_port is not None:
+            return self._mj_viewer
         else:
             return self.wrapped_env.viewer
 
-    def render(self, mode="human", **kwargs) -> Any:
-        if self._use_alt_viewer:
-            if self._alt_viewer is None:
-                from gym.envs.classic_control.rendering import SimpleImageViewer
-                import mujoco_py
+    def _setup_websocket_server(self) -> None:
+        import multiprocessing as mp
 
-                self._mj_viewer = mujoco_py.MjRenderContextOffscreen(
-                    self.wrapped_env.sim
-                )
-                self._alt_viewer = SimpleImageViewer()
+        self._mj_offscreen_viewer = mujoco_py.MjRenderContextOffscreen(
+            self.wrapped_env.sim
+        )
 
-            self._mj_viewer._set_mujoco_buffers()
-            self._mj_viewer.render(640, 480)
-            image = np.asarray(
-                self._mj_viewer.read_pixels(640, 480, depth=False)[::-1, :, :],
-                dtype=np.uint8,
-            )
-            if mode == "rgb_array":
-                return image
-            else:
-                if not (image.min() == 0 and image.max() == 0):
-                    self._alt_viewer.imshow(image)
-                return self._alt_viewer.isopen
+        class _ProcWorker(mp.Process):
+            def __init__(self, pipe: mp.connection.Pip) -> None:
+                super().__init__()
+                self.pipe = pipe
+
+            def _run_server(self) -> None:
+                import asyncio
+                import io
+                import websockets
+
+                async def handler(ws, _path):
+                    loop = asyncio.get_running_loop()
+                    while True:
+                        image = await loop.run_in_executor(None, self.pipe.recv)
+                        if image is None:
+                            print("Shutting down the websocket server...")
+                            return
+                        with io.BytesIO() as stream:
+                            image.save(stream, format="png")
+                            await ws.send(stream.getvalue())
+
+                server = websockets.serve(handler, "127.0.0.1", 5678)
+                asyncio.get_event_loop().run_until_complete(server)
+                asyncio.get_event_loop().run_forever()
+
+            def run(self) -> None:
+                try:
+                    self._run_server()
+                except KeyboardInterrupt:
+                    pass
+                except Exception as e:
+                    print("Exception in websocket server")
+                    raise e
+
+        self._websock_server_pipe, pipe = mp.Pipe()
+        worker = _ProcWorker(pipe)
+        worker.start()
+
+    def _render_image(self):
+        from PIL import Image
+
+        self._mj_offscreen_viewer._set_mujoco_buffers()
+        self._mj_offscreen_viewer.render(640, 480)
+        image_array = np.asarray(
+            self._mj_offscreen_viewer.read_pixels(640, 480, depth=False)[::-1, :, :],
+            dtype=np.uint8,
+        )
+        return Image.fromarray(image_array)
+
+    def render(self, *args, **kwargs) -> Any:
+        if self._websock_port is not None:
+            if self._mj_offscreen_viewer is None:
+                self._setup_websocket_viewer()
+            self._websock_server_pipe.send(self._render_image())
+            return None
         else:
-            return self.wrapped_env.render(mode=mode, **kwargs)
+            return self.wrapped_env.render(*args, **kwargs)
 
     @property
     def action_space(self):
@@ -450,8 +490,8 @@ class MazeEnv(gym.Env):
 
     def close(self) -> None:
         self.wrapped_env.close()
-        if self._alt_viewer is not None:
-            self._alt_viewer.close()
+        if self._websock_server_pipe is not None:
+            self._websock_server_pipe.send(self._render_image())
 
 
 def _add_object_ball(

From 15a1a1384f87e8f6002048d1ed1f3e50c2b1f47a Mon Sep 17 00:00:00 2001
From: kngwyu <yuji.kngw.80s.revive@gmail.com>
Date: Tue, 13 Apr 2021 19:18:50 +0900
Subject: [PATCH 05/24] Web-based viewer

---
 MANIFEST.in                   |  1 +
 mujoco_maze/maze_env.py       | 73 +++++++++++++++++++++--------------
 mujoco_maze/static/index.html | 27 +++++++++++++
 3 files changed, 73 insertions(+), 28 deletions(-)
 create mode 100644 MANIFEST.in
 create mode 100644 mujoco_maze/static/index.html

diff --git a/MANIFEST.in b/MANIFEST.in
new file mode 100644
index 0000000..fd381a6
--- /dev/null
+++ b/MANIFEST.in
@@ -0,0 +1 @@
+include mujoco-maze/static/index.html
\ No newline at end of file
diff --git a/mujoco_maze/maze_env.py b/mujoco_maze/maze_env.py
index 9f6d2e1..2542f67 100644
--- a/mujoco_maze/maze_env.py
+++ b/mujoco_maze/maze_env.py
@@ -33,7 +33,7 @@ class MazeEnv(gym.Env):
         inner_reward_scaling: float = 1.0,
         restitution_coef: float = 0.8,
         task_kwargs: dict = {},
-        websock_render_port: Optional[int] = None,
+        websock_port: Optional[int] = None,
         **kwargs,
     ) -> None:
         self._task = maze_task(maze_size_scaling, **task_kwargs)
@@ -362,37 +362,52 @@ class MazeEnv(gym.Env):
         else:
             return self.wrapped_env.viewer
 
-    def _setup_websocket_server(self) -> None:
+    def _setup_websock_server(self) -> None:
         import multiprocessing as mp
 
-        self._mj_offscreen_viewer = mujoco_py.MjRenderContextOffscreen(
-            self.wrapped_env.sim
-        )
-
         class _ProcWorker(mp.Process):
-            def __init__(self, pipe: mp.connection.Pip) -> None:
+            def __init__(self, pipe: mp.connection.Pipe, port: int) -> None:
                 super().__init__()
                 self.pipe = pipe
+                self.port = port
+                self.server = None
 
             def _run_server(self) -> None:
                 import asyncio
                 import io
-                import websockets
+                import fastapi
+                import pathlib
+                import uvicorn
 
-                async def handler(ws, _path):
+                from PIL import Image
+
+                app = fastapi.FastAPI()
+                html_path = pathlib.Path(__file__).parent.joinpath("static/index.html")
+                html = html_path.read_text().replace("{{port}}", str(self.port))
+
+                @app.get("/")
+                async def get():
+                    return fastapi.responses.HTMLResponse(html)
+
+                @app.websocket("/ws")
+                async def ws_send_image(websocket: fastapi.WebSocket):
+                    await websocket.accept()
                     loop = asyncio.get_running_loop()
                     while True:
-                        image = await loop.run_in_executor(None, self.pipe.recv)
-                        if image is None:
-                            print("Shutting down the websocket server...")
-                            return
+                        image_array = await loop.run_in_executor(None, self.pipe.recv)
+                        if image_array is None:
+                            break
+                        image = Image.fromarray(image_array)
                         with io.BytesIO() as stream:
                             image.save(stream, format="png")
-                            await ws.send(stream.getvalue())
+                            res = stream.getvalue()
+                            await websocket.send_bytes(res)
+                    await websocket.close()
+                    await self.server.shutdown()
 
-                server = websockets.serve(handler, "127.0.0.1", 5678)
-                asyncio.get_event_loop().run_until_complete(server)
-                asyncio.get_event_loop().run_forever()
+                config = uvicorn.Config(app, port=self.port)
+                self.server = uvicorn.Server(config)
+                self.server.run()
 
             def run(self) -> None:
                 try:
@@ -404,28 +419,30 @@ class MazeEnv(gym.Env):
                     raise e
 
         self._websock_server_pipe, pipe = mp.Pipe()
-        worker = _ProcWorker(pipe)
+        worker = _ProcWorker(pipe, self._websock_port)
         worker.start()
 
-    def _render_image(self):
-        from PIL import Image
-
+    def _render_image(self) -> np.ndarray:
         self._mj_offscreen_viewer._set_mujoco_buffers()
         self._mj_offscreen_viewer.render(640, 480)
-        image_array = np.asarray(
+        return np.asarray(
             self._mj_offscreen_viewer.read_pixels(640, 480, depth=False)[::-1, :, :],
             dtype=np.uint8,
         )
-        return Image.fromarray(image_array)
 
-    def render(self, *args, **kwargs) -> Any:
+    def render(self, mode="human", **kwargs) -> Any:
         if self._websock_port is not None:
             if self._mj_offscreen_viewer is None:
-                self._setup_websocket_viewer()
+                import mujoco_py
+
+                self._mj_offscreen_viewer = mujoco_py.MjRenderContextOffscreen(
+                    self.wrapped_env.sim
+                )
+                self._setup_websock_server()
             self._websock_server_pipe.send(self._render_image())
-            return None
+            return True
         else:
-            return self.wrapped_env.render(*args, **kwargs)
+            return self.wrapped_env.render(mode, **kwargs)
 
     @property
     def action_space(self):
@@ -491,7 +508,7 @@ class MazeEnv(gym.Env):
     def close(self) -> None:
         self.wrapped_env.close()
         if self._websock_server_pipe is not None:
-            self._websock_server_pipe.send(self._render_image())
+            self._websock_server_pipe.send(None)
 
 
 def _add_object_ball(
diff --git a/mujoco_maze/static/index.html b/mujoco_maze/static/index.html
new file mode 100644
index 0000000..ed58d17
--- /dev/null
+++ b/mujoco_maze/static/index.html
@@ -0,0 +1,27 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <title>MuJoCo maze visualizer</title>
+  </head>
+  <body>
+    <script>
+      var web_socket = new WebSocket('ws://127.0.0.1:{{port}}/ws');
+      web_socket.binaryType = "arraybuffer";
+      web_socket.onmessage = function(event) {
+          var canvas = document.getElementById('canvas');
+          var ctx = canvas.getContext('2d');
+          var blob = new Blob([event.data], {type:'image/png'});
+          var url = URL.createObjectURL(blob);
+          var image = new Image();
+          image.onload = function() {
+              ctx.drawImage(image, 0, 0);
+          }
+          console.log(url);
+          image.src = url;
+      }
+    </script>
+    <div>
+      <canvas id="canvas" width="600" height="480"></canvas>
+    </div>
+  </body>
+</html>

From a2ba3511cd61770c608b4e39a4c88e4910334846 Mon Sep 17 00:00:00 2001
From: kngwyu <yuji.kngw.80s.revive@gmail.com>
Date: Wed, 14 Apr 2021 12:06:56 +0900
Subject: [PATCH 06/24] Fix server shutdown

---
 MANIFEST.in                    |   3 ++-
 mujoco_maze/maze_env.py        |   8 ++++----
 mujoco_maze/static/favicon.ico | Bin 0 -> 2040 bytes
 3 files changed, 6 insertions(+), 5 deletions(-)
 create mode 100644 mujoco_maze/static/favicon.ico

diff --git a/MANIFEST.in b/MANIFEST.in
index fd381a6..67ae514 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -1 +1,2 @@
-include mujoco-maze/static/index.html
\ No newline at end of file
+include mujoco-maze/static/favicon.ico
+include mujoco-maze/templates/index.html
\ No newline at end of file
diff --git a/mujoco_maze/maze_env.py b/mujoco_maze/maze_env.py
index 2542f67..7e7a68f 100644
--- a/mujoco_maze/maze_env.py
+++ b/mujoco_maze/maze_env.py
@@ -370,7 +370,6 @@ class MazeEnv(gym.Env):
                 super().__init__()
                 self.pipe = pipe
                 self.port = port
-                self.server = None
 
             def _run_server(self) -> None:
                 import asyncio
@@ -384,6 +383,7 @@ class MazeEnv(gym.Env):
                 app = fastapi.FastAPI()
                 html_path = pathlib.Path(__file__).parent.joinpath("static/index.html")
                 html = html_path.read_text().replace("{{port}}", str(self.port))
+                server = None
 
                 @app.get("/")
                 async def get():
@@ -403,11 +403,11 @@ class MazeEnv(gym.Env):
                             res = stream.getvalue()
                             await websocket.send_bytes(res)
                     await websocket.close()
-                    await self.server.shutdown()
+                    server.should_exit = True
 
                 config = uvicorn.Config(app, port=self.port)
-                self.server = uvicorn.Server(config)
-                self.server.run()
+                server = uvicorn.Server(config)
+                server.run()
 
             def run(self) -> None:
                 try:
diff --git a/mujoco_maze/static/favicon.ico b/mujoco_maze/static/favicon.ico
new file mode 100644
index 0000000000000000000000000000000000000000..45acf83b6535edeae3c9da3588404583625b926b
GIT binary patch
literal 2040
zcmV<U2M73xP)<h;3K|Lk000e1NJLTq001BW001Be1^@s6b9#F800009a7bBm000ie
z000ie0hKEb8vp<Ys!2paR9J<LmrIW%M-_#?8xi@aM^<%JSHEi7NTU%RFa{&Ak-Y-%
zL1GIM5`Ta{!QKltEWKgJ3dD+^fP@hOd*eaEGt*t&_3nD-GlE4{XZLu<W@1&kbu;4R
zx#yl6Dc}0puN{C=3K79MhqV>}tqtHPiUI&73Q=Fq(EwO$c~md>@V$3<)p|Z#@z;Mp
z#W{x{D6PspopmH}g0+?~4DmdlBu+@;1mE`v!w_pNNu1ztgkgv=rp!rFf*>FaL-H&q
zPP11Y4$y7X$kL1?PEcAAh9M#f^WBGiloHnKHEEKfd`+!(LQxc44Xyw<IX%JmedhBy
z!|Neo7;<`cinW%(<pAd#y`vsUJmtxgC$9wH95fm=>WwDb&6eTy5D`H>`sgEfKk?$?
zl0W_H0xt|{Hk;t!W^n@!>h(H);FF~po6QCU+U+(X!e+f889X7_PG3!d(wN5|u-ons
z5nAmwB4ECpHy)3$X^hlr*uvs)D5b!G!<8}TD$Q7H5d;yz_yKHReZF`>iKBY1K#-RK
zj7B4(#sP&=OePbw(e(R$oO6tZBNpj`dZW(K@zDz<=yW?geCJ&v57x^S&N-S<6H$tI
zw_`ZH;cB&}-#<e{7>!0Gaq=>N$z;Mhs?)5sIXXTf&vS;uAplVnp|m1RQij7Jg3#+7
zf(Y?0#sv+|zx@`CW&`IOtL2I$O;Jiwn@&hS|D4Hqf_0X_4`{Vom`^|b^aTJ{*F$QD
zkI`D8wI<JVY+?6hEMo}wVj2-CX-2TlVvH%<RM?jYvMjr6*}_s-i`E)>{P^(;B3vao
z4}bD=o<DnzwU%boq|@n?bPonN>xfzr2L}f)l%U;i(>?4WB310*;DD$Vk>@$jo<7Ig
zf`d+nC~A^rIoDTLF9S%u2F*9$CC~19TIo`Rx8{RT3A5P0p@{5Fib`Oj6e48{+M+;2
z%IEII%r}1cW1?1s5@EgGFkSDMZZ;erA0Z-4CsS6d6<TY~_5zyE=PYg(hzR`$eUwrx
zmn#-GHvpXV&oIWY*=(3iX8;_ZoDc*)aU3%qjRENOddRPS`+Mh{Lu*YGMJyK!KKaKd
zBuNTDtrnt`!WNcYyaR;5_c5LU2ixry2YANd2R;D1U5vFBtrTG`ESXJGvOEW%(WnEE
zXE`RzGO{$q^E{4^kI;L%<J}H`gM&84GbC|JJfByfYV{h<I%cz3=?jepwR)}8(Q-wa
zrWj-BbUPp=fZIL0-404CcH0<v@0Y)EN-455W4T%a5QZT^P#VW<S{~s0K2a+w4_K{8
zlN1r5)9oN4BuUD8wJO`S72*3n&N*hYIXFiU1T>>2rbzhU&GS-7IE>!6QJSU1ySUt-
zS#D4i1=(JVzVGAvCB@cS@;paG@H`Kt6nUPL?<oeU02F!7ZHax~=jQ2uIr+h(YA<s&
zxUNv2?HzAAnPO*iw9%aP`v7b=8wQtGRarWnF4kIx*TagTv;Nthp*4fcK`FGO9>y4o
zqF``2V14n7*=)w?>1o+vMNv=`1)k?&j6v)2ZHi*=)y9;@;>vqLQ4}bxt9Vh^GHz`Z
z6Ok$xz!-z)d01<)dB%7=u3FrjoSl@lp3Ew$j!%yPNaCa#$&F@%+uX%`!F)DHL^wJr
z+cnSfDmMs2PEO0*^?J>GR*JWKc!;+-dl7)|`{a3E)mj^k=XuE9-*4-xwW{it7kBl(
z)c|l8v$o0=QB_<78l4{F@v}Ps<Ixyn3`fVk(zDZyo7t=q-tln{fH;nsjwe{>_Qs`f
zlzIVBA{_S0+@i=Ck0*QjHHW>!lF=k3%PgzWr~+l;T}-Rh<QMNfLM!3x)?fbi8UGor
z=yW^Pz?n^EdxkXK-XVezCownk(iLm98uiYt@hr*EEEY>X{`lh(K(pDPT@U~7;Q%(z
zXfzvaH{1OLbTpccJ-xzuwJ!6lrO|Ab%qU^C_Xq1N&8S7*?%b82bJ!)~t1pxHmn$+l
z-5!_EFLAD%Vdrn0BTBG^<?`Zkf2g8=-bWC!EaUpB6sPe#9z5t%|K?k`cukh37;7!<
zy7yWDk|aT_tL8s|!{V&qcrk9REvKJWh^R8ND6rP@_J=>k^E@suE-}|v*S!D!cU}v?
zY`!E(2K3J#RNoMTiwm3sglaN6=a>yH%Imz-oIN;0D~-c3ydKg!?BSf_<n#pN8IN}D
zj)D7caerBB$@az~B2}Kdy-m2gxV9;O{N6ulhKfIY(PCh7KI0*{@&@(A7f*4{vfivQ
zLEzK$%~u{=QSBVs52!~?mdj-oza3D(o!B-v{C&OTDDwF8(^JxeI^r#W(iQWfY{Bt(
zjEO=Iv;R!`x|QcSwMLVA6mhlpd4Th8yitK4TwGEV1seF>;1RWYo!(pDXE2<yT$G?&
z?H28WHn`FyO%%Mo0JY~=>aW&4;%<QyDEe>z2#m)nu^@LZSWQd;&-XB=hppEJaI;tv
z;8#8mFuEE6C8+j68*R#){<HClG)VyP1HYOtMP4wx8dlGnZnO5<0CEeaC<wx!il=G1
z|K-ryR2P};W?TKIwXWiYE|{iwpKmG(Uivn9<%;7N(O)N4d7(V&o~~}k@@@S79{&Rt
WQViuANLz~l0000<MNUMnLSTY7wAEq&

literal 0
HcmV?d00001


From 62188b5867fae160e3c4efdf35b86b71900b6974 Mon Sep 17 00:00:00 2001
From: kngwyu <yuji.kngw.80s.revive@gmail.com>
Date: Mon, 19 Apr 2021 18:42:06 +0900
Subject: [PATCH 07/24] Fix reacher link

---
 mujoco_maze/reacher.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/mujoco_maze/reacher.py b/mujoco_maze/reacher.py
index 7a97118..d94cb4a 100644
--- a/mujoco_maze/reacher.py
+++ b/mujoco_maze/reacher.py
@@ -1,7 +1,7 @@
 """
 Based on the reacher in `dm_control`_.
 
-.. _gym: https://github.com/openai/gym
+.. _dm_control: https://github.com/deepmind/dm_control
 """
 
 from typing import Tuple
@@ -54,10 +54,14 @@ class ReacherEnv(AgentModel):
 
     def reset_model(self) -> np.ndarray:
         qpos = self.init_qpos + self.np_random.uniform(
-            low=-0.1, high=0.1, size=self.model.nq,
+            low=-0.1,
+            high=0.1,
+            size=self.model.nq,
         )
         qvel = self.init_qvel + self.np_random.uniform(
-            low=-0.1, high=0.1, size=self.model.nv,
+            low=-0.1,
+            high=0.1,
+            size=self.model.nv,
         )
 
         self.set_state(qpos, qvel)

From 22d7f419fc74e849d5195258a1a64467d0f27403 Mon Sep 17 00:00:00 2001
From: kngwyu <yuji.kngw.80s.revive@gmail.com>
Date: Mon, 19 Apr 2021 20:11:35 +0900
Subject: [PATCH 08/24] Spearate websock_viewer file

---
 mujoco_maze/maze_env.py        |  70 +++------------------------------
 mujoco_maze/static/favicon.ico | Bin 2040 -> 0 bytes
 mujoco_maze/websock_viewer.py  |  65 ++++++++++++++++++++++++++++++
 3 files changed, 70 insertions(+), 65 deletions(-)
 delete mode 100644 mujoco_maze/static/favicon.ico
 create mode 100644 mujoco_maze/websock_viewer.py

diff --git a/mujoco_maze/maze_env.py b/mujoco_maze/maze_env.py
index 7e7a68f..3a455d0 100644
--- a/mujoco_maze/maze_env.py
+++ b/mujoco_maze/maze_env.py
@@ -362,66 +362,6 @@ class MazeEnv(gym.Env):
         else:
             return self.wrapped_env.viewer
 
-    def _setup_websock_server(self) -> None:
-        import multiprocessing as mp
-
-        class _ProcWorker(mp.Process):
-            def __init__(self, pipe: mp.connection.Pipe, port: int) -> None:
-                super().__init__()
-                self.pipe = pipe
-                self.port = port
-
-            def _run_server(self) -> None:
-                import asyncio
-                import io
-                import fastapi
-                import pathlib
-                import uvicorn
-
-                from PIL import Image
-
-                app = fastapi.FastAPI()
-                html_path = pathlib.Path(__file__).parent.joinpath("static/index.html")
-                html = html_path.read_text().replace("{{port}}", str(self.port))
-                server = None
-
-                @app.get("/")
-                async def get():
-                    return fastapi.responses.HTMLResponse(html)
-
-                @app.websocket("/ws")
-                async def ws_send_image(websocket: fastapi.WebSocket):
-                    await websocket.accept()
-                    loop = asyncio.get_running_loop()
-                    while True:
-                        image_array = await loop.run_in_executor(None, self.pipe.recv)
-                        if image_array is None:
-                            break
-                        image = Image.fromarray(image_array)
-                        with io.BytesIO() as stream:
-                            image.save(stream, format="png")
-                            res = stream.getvalue()
-                            await websocket.send_bytes(res)
-                    await websocket.close()
-                    server.should_exit = True
-
-                config = uvicorn.Config(app, port=self.port)
-                server = uvicorn.Server(config)
-                server.run()
-
-            def run(self) -> None:
-                try:
-                    self._run_server()
-                except KeyboardInterrupt:
-                    pass
-                except Exception as e:
-                    print("Exception in websocket server")
-                    raise e
-
-        self._websock_server_pipe, pipe = mp.Pipe()
-        worker = _ProcWorker(pipe, self._websock_port)
-        worker.start()
-
     def _render_image(self) -> np.ndarray:
         self._mj_offscreen_viewer._set_mujoco_buffers()
         self._mj_offscreen_viewer.render(640, 480)
@@ -433,12 +373,12 @@ class MazeEnv(gym.Env):
     def render(self, mode="human", **kwargs) -> Any:
         if self._websock_port is not None:
             if self._mj_offscreen_viewer is None:
-                import mujoco_py
+                from mujoco_py import MjRenderContextOffscreen as MjRenderOffscreen
 
-                self._mj_offscreen_viewer = mujoco_py.MjRenderContextOffscreen(
-                    self.wrapped_env.sim
-                )
-                self._setup_websock_server()
+                from mujoco_maze.websock_viewer import start_server
+
+                self._mj_offscreen_viewer = MjRenderOffscreen(self.wrapped_env.sim)
+                self._websock_server_pipe = start_server(self._websock_port)
             self._websock_server_pipe.send(self._render_image())
             return True
         else:
diff --git a/mujoco_maze/static/favicon.ico b/mujoco_maze/static/favicon.ico
deleted file mode 100644
index 45acf83b6535edeae3c9da3588404583625b926b..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 2040
zcmV<U2M73xP)<h;3K|Lk000e1NJLTq001BW001Be1^@s6b9#F800009a7bBm000ie
z000ie0hKEb8vp<Ys!2paR9J<LmrIW%M-_#?8xi@aM^<%JSHEi7NTU%RFa{&Ak-Y-%
zL1GIM5`Ta{!QKltEWKgJ3dD+^fP@hOd*eaEGt*t&_3nD-GlE4{XZLu<W@1&kbu;4R
zx#yl6Dc}0puN{C=3K79MhqV>}tqtHPiUI&73Q=Fq(EwO$c~md>@V$3<)p|Z#@z;Mp
z#W{x{D6PspopmH}g0+?~4DmdlBu+@;1mE`v!w_pNNu1ztgkgv=rp!rFf*>FaL-H&q
zPP11Y4$y7X$kL1?PEcAAh9M#f^WBGiloHnKHEEKfd`+!(LQxc44Xyw<IX%JmedhBy
z!|Neo7;<`cinW%(<pAd#y`vsUJmtxgC$9wH95fm=>WwDb&6eTy5D`H>`sgEfKk?$?
zl0W_H0xt|{Hk;t!W^n@!>h(H);FF~po6QCU+U+(X!e+f889X7_PG3!d(wN5|u-ons
z5nAmwB4ECpHy)3$X^hlr*uvs)D5b!G!<8}TD$Q7H5d;yz_yKHReZF`>iKBY1K#-RK
zj7B4(#sP&=OePbw(e(R$oO6tZBNpj`dZW(K@zDz<=yW?geCJ&v57x^S&N-S<6H$tI
zw_`ZH;cB&}-#<e{7>!0Gaq=>N$z;Mhs?)5sIXXTf&vS;uAplVnp|m1RQij7Jg3#+7
zf(Y?0#sv+|zx@`CW&`IOtL2I$O;Jiwn@&hS|D4Hqf_0X_4`{Vom`^|b^aTJ{*F$QD
zkI`D8wI<JVY+?6hEMo}wVj2-CX-2TlVvH%<RM?jYvMjr6*}_s-i`E)>{P^(;B3vao
z4}bD=o<DnzwU%boq|@n?bPonN>xfzr2L}f)l%U;i(>?4WB310*;DD$Vk>@$jo<7Ig
zf`d+nC~A^rIoDTLF9S%u2F*9$CC~19TIo`Rx8{RT3A5P0p@{5Fib`Oj6e48{+M+;2
z%IEII%r}1cW1?1s5@EgGFkSDMZZ;erA0Z-4CsS6d6<TY~_5zyE=PYg(hzR`$eUwrx
zmn#-GHvpXV&oIWY*=(3iX8;_ZoDc*)aU3%qjRENOddRPS`+Mh{Lu*YGMJyK!KKaKd
zBuNTDtrnt`!WNcYyaR;5_c5LU2ixry2YANd2R;D1U5vFBtrTG`ESXJGvOEW%(WnEE
zXE`RzGO{$q^E{4^kI;L%<J}H`gM&84GbC|JJfByfYV{h<I%cz3=?jepwR)}8(Q-wa
zrWj-BbUPp=fZIL0-404CcH0<v@0Y)EN-455W4T%a5QZT^P#VW<S{~s0K2a+w4_K{8
zlN1r5)9oN4BuUD8wJO`S72*3n&N*hYIXFiU1T>>2rbzhU&GS-7IE>!6QJSU1ySUt-
zS#D4i1=(JVzVGAvCB@cS@;paG@H`Kt6nUPL?<oeU02F!7ZHax~=jQ2uIr+h(YA<s&
zxUNv2?HzAAnPO*iw9%aP`v7b=8wQtGRarWnF4kIx*TagTv;Nthp*4fcK`FGO9>y4o
zqF``2V14n7*=)w?>1o+vMNv=`1)k?&j6v)2ZHi*=)y9;@;>vqLQ4}bxt9Vh^GHz`Z
z6Ok$xz!-z)d01<)dB%7=u3FrjoSl@lp3Ew$j!%yPNaCa#$&F@%+uX%`!F)DHL^wJr
z+cnSfDmMs2PEO0*^?J>GR*JWKc!;+-dl7)|`{a3E)mj^k=XuE9-*4-xwW{it7kBl(
z)c|l8v$o0=QB_<78l4{F@v}Ps<Ixyn3`fVk(zDZyo7t=q-tln{fH;nsjwe{>_Qs`f
zlzIVBA{_S0+@i=Ck0*QjHHW>!lF=k3%PgzWr~+l;T}-Rh<QMNfLM!3x)?fbi8UGor
z=yW^Pz?n^EdxkXK-XVezCownk(iLm98uiYt@hr*EEEY>X{`lh(K(pDPT@U~7;Q%(z
zXfzvaH{1OLbTpccJ-xzuwJ!6lrO|Ab%qU^C_Xq1N&8S7*?%b82bJ!)~t1pxHmn$+l
z-5!_EFLAD%Vdrn0BTBG^<?`Zkf2g8=-bWC!EaUpB6sPe#9z5t%|K?k`cukh37;7!<
zy7yWDk|aT_tL8s|!{V&qcrk9REvKJWh^R8ND6rP@_J=>k^E@suE-}|v*S!D!cU}v?
zY`!E(2K3J#RNoMTiwm3sglaN6=a>yH%Imz-oIN;0D~-c3ydKg!?BSf_<n#pN8IN}D
zj)D7caerBB$@az~B2}Kdy-m2gxV9;O{N6ulhKfIY(PCh7KI0*{@&@(A7f*4{vfivQ
zLEzK$%~u{=QSBVs52!~?mdj-oza3D(o!B-v{C&OTDDwF8(^JxeI^r#W(iQWfY{Bt(
zjEO=Iv;R!`x|QcSwMLVA6mhlpd4Th8yitK4TwGEV1seF>;1RWYo!(pDXE2<yT$G?&
z?H28WHn`FyO%%Mo0JY~=>aW&4;%<QyDEe>z2#m)nu^@LZSWQd;&-XB=hppEJaI;tv
z;8#8mFuEE6C8+j68*R#){<HClG)VyP1HYOtMP4wx8dlGnZnO5<0CEeaC<wx!il=G1
z|K-ryR2P};W?TKIwXWiYE|{iwpKmG(Uivn9<%;7N(O)N4d7(V&o~~}k@@@S79{&Rt
WQViuANLz~l0000<MNUMnLSTY7wAEq&

diff --git a/mujoco_maze/websock_viewer.py b/mujoco_maze/websock_viewer.py
new file mode 100644
index 0000000..9dad232
--- /dev/null
+++ b/mujoco_maze/websock_viewer.py
@@ -0,0 +1,65 @@
+import asyncio
+import io
+import multiprocessing as mp
+import pathlib
+
+import fastapi
+import uvicorn
+
+from PIL import Image
+
+
+class _ServerWorker(mp.Process):
+    def __init__(self, pipe: mp.connection.Pipe, port: int) -> None:
+        super().__init__()
+        self.pipe = pipe
+        self.port = port
+
+    def _run_server(self) -> None:
+
+        app = fastapi.FastAPI()
+        static = pathlib.Path(__file__).parent.joinpath("static")
+        html_path = static.joinpath("index.html")
+        html = html_path.read_text().replace("{{port}}", str(self.port))
+
+        @app.get("/")
+        async def get():
+            return fastapi.responses.HTMLResponse(html)
+
+        server = None
+
+        @app.websocket("/ws")
+        async def ws_send_image(websocket: fastapi.WebSocket):
+            await websocket.accept()
+            loop = asyncio.get_running_loop()
+            while True:
+                image_array = await loop.run_in_executor(None, self.pipe.recv)
+                if image_array is None:
+                    break
+                image = Image.fromarray(image_array)
+                with io.BytesIO() as stream:
+                    image.save(stream, format="png")
+                    res = stream.getvalue()
+                    await websocket.send_bytes(res)
+            await websocket.close()
+            server.should_exit = True
+
+        config = uvicorn.Config(app, port=self.port)
+        server = uvicorn.Server(config)
+        server.run()
+
+    def run(self) -> None:
+        try:
+            self._run_server()
+        except KeyboardInterrupt:
+            pass
+        except Exception as e:
+            print("Exception in websocket server")
+            raise e
+
+
+def start_server(port: int) -> mp.connection.Connection:
+    mainproc_pipe, server_pipe = mp.Pipe()
+    worker = _ServerWorker(server_pipe, port)
+    worker.start()
+    return mainproc_pipe

From e1af62cdc7db01f68445cd50f284627246d93d12 Mon Sep 17 00:00:00 2001
From: kngwyu <yuji.kngw.80s.revive@gmail.com>
Date: Mon, 19 Apr 2021 20:15:33 +0900
Subject: [PATCH 09/24] Use Python string for HTML

---
 mujoco_maze/static/index.html | 27 ---------------------------
 mujoco_maze/websock_viewer.py | 35 +++++++++++++++++++++++++++++++----
 2 files changed, 31 insertions(+), 31 deletions(-)
 delete mode 100644 mujoco_maze/static/index.html

diff --git a/mujoco_maze/static/index.html b/mujoco_maze/static/index.html
deleted file mode 100644
index ed58d17..0000000
--- a/mujoco_maze/static/index.html
+++ /dev/null
@@ -1,27 +0,0 @@
-<!DOCTYPE html>
-<html>
-  <head>
-    <title>MuJoCo maze visualizer</title>
-  </head>
-  <body>
-    <script>
-      var web_socket = new WebSocket('ws://127.0.0.1:{{port}}/ws');
-      web_socket.binaryType = "arraybuffer";
-      web_socket.onmessage = function(event) {
-          var canvas = document.getElementById('canvas');
-          var ctx = canvas.getContext('2d');
-          var blob = new Blob([event.data], {type:'image/png'});
-          var url = URL.createObjectURL(blob);
-          var image = new Image();
-          image.onload = function() {
-              ctx.drawImage(image, 0, 0);
-          }
-          console.log(url);
-          image.src = url;
-      }
-    </script>
-    <div>
-      <canvas id="canvas" width="600" height="480"></canvas>
-    </div>
-  </body>
-</html>
diff --git a/mujoco_maze/websock_viewer.py b/mujoco_maze/websock_viewer.py
index 9dad232..07b82d0 100644
--- a/mujoco_maze/websock_viewer.py
+++ b/mujoco_maze/websock_viewer.py
@@ -1,13 +1,42 @@
 import asyncio
 import io
 import multiprocessing as mp
-import pathlib
 
 import fastapi
 import uvicorn
 
 from PIL import Image
 
+HTML = """
+<!DOCTYPE html>
+<html>
+  <head>
+    <title>MuJoCo maze visualizer</title>
+  </head>
+  <body>
+    <script>
+      var web_socket = new WebSocket('ws://127.0.0.1:{{port}}/ws');
+      web_socket.binaryType = "arraybuffer";
+      web_socket.onmessage = function(event) {
+          var canvas = document.getElementById('canvas');
+          var ctx = canvas.getContext('2d');
+          var blob = new Blob([event.data], {type:'image/png'});
+          var url = URL.createObjectURL(blob);
+          var image = new Image();
+          image.onload = function() {
+              ctx.drawImage(image, 0, 0);
+          }
+          console.log(url);
+          image.src = url;
+      }
+    </script>
+    <div>
+      <canvas id="canvas" width="600" height="480"></canvas>
+    </div>
+  </body>
+</html>
+"""
+
 
 class _ServerWorker(mp.Process):
     def __init__(self, pipe: mp.connection.Pipe, port: int) -> None:
@@ -18,9 +47,7 @@ class _ServerWorker(mp.Process):
     def _run_server(self) -> None:
 
         app = fastapi.FastAPI()
-        static = pathlib.Path(__file__).parent.joinpath("static")
-        html_path = static.joinpath("index.html")
-        html = html_path.read_text().replace("{{port}}", str(self.port))
+        html = HTML.replace("{{port}}", str(self.port))
 
         @app.get("/")
         async def get():

From 4087203b0688414844e6999567a4d36f9b7accbe Mon Sep 17 00:00:00 2001
From: kngwyu <yuji.kngw.80s.revive@gmail.com>
Date: Tue, 20 Apr 2021 10:09:14 +0900
Subject: [PATCH 10/24] Formatting

---
 mujoco_maze/agent_model.py    | 9 +++------
 mujoco_maze/ant.py            | 4 +++-
 mujoco_maze/maze_env.py       | 9 ++++-----
 mujoco_maze/maze_env_utils.py | 7 ++++---
 mujoco_maze/swimmer.py        | 8 ++++++--
 mujoco_maze/websock_viewer.py | 1 -
 6 files changed, 20 insertions(+), 18 deletions(-)

diff --git a/mujoco_maze/agent_model.py b/mujoco_maze/agent_model.py
index d209d78..9d50fe4 100644
--- a/mujoco_maze/agent_model.py
+++ b/mujoco_maze/agent_model.py
@@ -27,16 +27,13 @@ class AgentModel(ABC, MujocoEnv, EzPickle):
 
     @abstractmethod
     def _get_obs(self) -> np.ndarray:
-        """Returns the observation from the model.
-        """
+        """Returns the observation from the model."""
         pass
 
     def get_xy(self) -> np.ndarray:
-        """Returns the coordinate of the agent.
-        """
+        """Returns the coordinate of the agent."""
         pass
 
     def set_xy(self, xy: np.ndarray) -> None:
-        """Set the coordinate of the agent.
-        """
+        """Set the coordinate of the agent."""
         pass
diff --git a/mujoco_maze/ant.py b/mujoco_maze/ant.py
index 34356e1..01a9e20 100644
--- a/mujoco_maze/ant.py
+++ b/mujoco_maze/ant.py
@@ -82,7 +82,9 @@ class AntEnv(AgentModel):
 
     def reset_model(self):
         qpos = self.init_qpos + self.np_random.uniform(
-            size=self.model.nq, low=-0.1, high=0.1,
+            size=self.model.nq,
+            low=-0.1,
+            high=0.1,
         )
         qvel = self.init_qvel + self.np_random.randn(self.model.nv) * 0.1
 
diff --git a/mujoco_maze/maze_env.py b/mujoco_maze/maze_env.py
index 3a455d0..6b89d87 100644
--- a/mujoco_maze/maze_env.py
+++ b/mujoco_maze/maze_env.py
@@ -370,17 +370,16 @@ class MazeEnv(gym.Env):
             dtype=np.uint8,
         )
 
-    def render(self, mode="human", **kwargs) -> Any:
-        if self._websock_port is not None:
+    def render(self, mode="human", **kwargs) -> Optional[np.ndarray]:
+        if mode == "human" and self._websock_port is not None:
             if self._mj_offscreen_viewer is None:
-                from mujoco_py import MjRenderContextOffscreen as MjRenderOffscreen
+                from mujoco_py import MjRenderContextOffscreen as MjRCO
 
                 from mujoco_maze.websock_viewer import start_server
 
-                self._mj_offscreen_viewer = MjRenderOffscreen(self.wrapped_env.sim)
+                self._mj_offscreen_viewer = MjRCO(self.wrapped_env.sim)
                 self._websock_server_pipe = start_server(self._websock_port)
             self._websock_server_pipe.send(self._render_image())
-            return True
         else:
             return self.wrapped_env.render(mode, **kwargs)
 
diff --git a/mujoco_maze/maze_env_utils.py b/mujoco_maze/maze_env_utils.py
index 348b88c..f9edfe1 100644
--- a/mujoco_maze/maze_env_utils.py
+++ b/mujoco_maze/maze_env_utils.py
@@ -83,7 +83,9 @@ class MazeCell(Enum):
 
 class Line:
     def __init__(
-        self, p1: Union[Sequence[float], Point], p2: Union[Sequence[float], Point],
+        self,
+        p1: Union[Sequence[float], Point],
+        p2: Union[Sequence[float], Point],
     ) -> None:
         self.p1 = p1 if isinstance(p1, Point) else np.complex(*p1)
         self.p2 = p2 if isinstance(p2, Point) else np.complex(*p2)
@@ -141,8 +143,7 @@ class Collision:
 
 
 class CollisionDetector:
-    """For manual collision detection.
-    """
+    """For manual collision detection."""
 
     EPS: float = 0.05
     NEIGHBORS: List[Tuple[int, int]] = [[0, -1], [-1, 0], [0, 1], [1, 0]]
diff --git a/mujoco_maze/swimmer.py b/mujoco_maze/swimmer.py
index ef56e89..7d2c6de 100644
--- a/mujoco_maze/swimmer.py
+++ b/mujoco_maze/swimmer.py
@@ -55,10 +55,14 @@ class SwimmerEnv(AgentModel):
 
     def reset_model(self) -> np.ndarray:
         qpos = self.init_qpos + self.np_random.uniform(
-            low=-0.1, high=0.1, size=self.model.nq,
+            low=-0.1,
+            high=0.1,
+            size=self.model.nq,
         )
         qvel = self.init_qvel + self.np_random.uniform(
-            low=-0.1, high=0.1, size=self.model.nv,
+            low=-0.1,
+            high=0.1,
+            size=self.model.nv,
         )
 
         self.set_state(qpos, qvel)
diff --git a/mujoco_maze/websock_viewer.py b/mujoco_maze/websock_viewer.py
index 07b82d0..a355b3f 100644
--- a/mujoco_maze/websock_viewer.py
+++ b/mujoco_maze/websock_viewer.py
@@ -4,7 +4,6 @@ import multiprocessing as mp
 
 import fastapi
 import uvicorn
-
 from PIL import Image
 
 HTML = """

From 409ee44568579633894f88bd3f6c222cd5d3157e Mon Sep 17 00:00:00 2001
From: kngwyu <yuji.kngw.80s.revive@gmail.com>
Date: Fri, 7 May 2021 19:10:19 +0900
Subject: [PATCH 11/24] Refactor maze_env

---
 mujoco_maze/maze_env.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/mujoco_maze/maze_env.py b/mujoco_maze/maze_env.py
index 6b89d87..d78b3bb 100644
--- a/mujoco_maze/maze_env.py
+++ b/mujoco_maze/maze_env.py
@@ -27,7 +27,7 @@ class MazeEnv(gym.Env):
         self,
         model_cls: Type[AgentModel],
         maze_task: Type[maze_task.MazeTask] = maze_task.MazeTask,
-        top_down_view: float = False,
+        include_position: bool = True,
         maze_height: float = 0.5,
         maze_size_scaling: float = 4.0,
         inner_reward_scaling: float = 1.0,
@@ -36,16 +36,11 @@ class MazeEnv(gym.Env):
         websock_port: Optional[int] = None,
         **kwargs,
     ) -> None:
+        self.t = 0  # time steps
         self._task = maze_task(maze_size_scaling, **task_kwargs)
-
-        xml_path = os.path.join(MODEL_DIR, model_cls.FILE)
-        tree = ET.parse(xml_path)
-        worldbody = tree.find(".//worldbody")
-
         self._maze_height = height = maze_height
         self._maze_size_scaling = size_scaling = maze_size_scaling
         self._inner_reward_scaling = inner_reward_scaling
-        self.t = 0  # time steps
         self._observe_blocks = self._task.OBSERVE_BLOCKS
         self._put_spin_near_agent = self._task.PUT_SPIN_NEAR_AGENT
         # Observe other objectives
@@ -94,6 +89,11 @@ class MazeEnv(gym.Env):
         # walls (immovable), chasms (fall), movable blocks
         self._view = np.zeros([5, 5, 3])
 
+        # Let's create MuJoCo XML
+        xml_path = os.path.join(MODEL_DIR, model_cls.FILE)
+        tree = ET.parse(xml_path)
+        worldbody = tree.find(".//worldbody")
+
         height_offset = 0.0
         if self.elevated:
             # Increase initial z-pos of ant.

From d043a35e6c90dfe5b97fc05c8ac85557c0bdd08f Mon Sep 17 00:00:00 2001
From: kngwyu <yuji.kngw.80s.revive@gmail.com>
Date: Fri, 7 May 2021 19:25:57 +0900
Subject: [PATCH 12/24] Make ORI_IND optional and adjust the size of maze for
 swimmer

---
 mujoco_maze/agent_model.py | 2 +-
 mujoco_maze/maze_task.py   | 2 +-
 mujoco_maze/swimmer.py     | 1 -
 3 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/mujoco_maze/agent_model.py b/mujoco_maze/agent_model.py
index 9d50fe4..c749639 100644
--- a/mujoco_maze/agent_model.py
+++ b/mujoco_maze/agent_model.py
@@ -11,7 +11,7 @@ from gym.utils import EzPickle
 class AgentModel(ABC, MujocoEnv, EzPickle):
     FILE: str
     MANUAL_COLLISION: bool
-    ORI_IND: int
+    ORI_IND: Optional[int] = None
     RADIUS: Optional[float] = None
 
     def __init__(self, file_path: str, frame_skip: int) -> None:
diff --git a/mujoco_maze/maze_task.py b/mujoco_maze/maze_task.py
index 828e140..09666c3 100644
--- a/mujoco_maze/maze_task.py
+++ b/mujoco_maze/maze_task.py
@@ -331,7 +331,7 @@ class SubGoalTRoom(GoalRewardTRoom):
 
 class NoRewardRoom(MazeTask):
     REWARD_THRESHOLD: float = 0.0
-    MAZE_SIZE_SCALING: Scaling = Scaling(4.0, 4.0, 4.0)
+    MAZE_SIZE_SCALING: Scaling = Scaling(4.0, 4.0, 1.0)
 
     def reward(self, obs: np.ndarray) -> float:
         return 0.0
diff --git a/mujoco_maze/swimmer.py b/mujoco_maze/swimmer.py
index 7d2c6de..efe225a 100644
--- a/mujoco_maze/swimmer.py
+++ b/mujoco_maze/swimmer.py
@@ -37,7 +37,6 @@ class SwimmerEnv(AgentModel):
     def step(self, action: np.ndarray) -> Tuple[np.ndarray, float, bool, dict]:
         xy_pos_before = self.sim.data.qpos[:2].copy()
         self.do_simulation(action, self.frame_skip)
-
         forward_reward = self._forward_reward(xy_pos_before)
         ctrl_cost = self._ctrl_cost_weight * np.sum(np.square(action))
         return (

From 053e74f04c77b04bb385161fdc5a8b65e9cf6105 Mon Sep 17 00:00:00 2001
From: Yuji Kanagawa <yuji-kanagawa@saion-login1.oist.jp>
Date: Mon, 10 May 2021 16:59:46 +0900
Subject: [PATCH 13/24] Fix some typos in README

---
 README.md | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/README.md b/README.md
index 51731d2..dcddfb9 100644
--- a/README.md
+++ b/README.md
@@ -9,14 +9,15 @@ and [openai gym][gym].
 Thankfully, this project is based on the code from  [rllab] and
 [tensorflow/models][models].
 
-Note that [d4rl][d4rl] and [dm_control][dm_control] have similar maze
+Note that [d4rl] and [dm_control] have similar maze
 environments, and you can also check them.
 But, if you want more customizable or minimal one, I recommend this.
 
 ## Usage
 
-Importing `mujoco_maze` registers environments and environments listed
-below are available via `gym.make`.
+Importing `mujoco_maze` registers environments and you can load
+environments by `gym.make`.
+All available environments listed are listed in [Environments] section.
 
 E.g.,:
 ```python
@@ -116,7 +117,7 @@ not tested well.
 
 ## License
 This project is licensed under Apache License, Version 2.0
-([LICENSE-APACHE](LICENSE) or http://www.apache.org/licenses/LICENSE-2.0).
+([LICENSE](LICENSE) or http://www.apache.org/licenses/LICENSE-2.0).
 
 [d4rl]: https://github.com/rail-berkeley/d4rl
 [dm_control]: https://github.com/deepmind/dm_control

From 002688573096b4d9c877e57a735e9da0e808667f Mon Sep 17 00:00:00 2001
From: kngwyu <yuji.kanagawa@oist.jp>
Date: Sat, 15 May 2021 19:59:12 +0900
Subject: [PATCH 14/24] Rename NoRewardRoom NoReWardCorridor

---
 mujoco_maze/maze_task.py | 23 +++++++++++++++++++++--
 1 file changed, 21 insertions(+), 2 deletions(-)

diff --git a/mujoco_maze/maze_task.py b/mujoco_maze/maze_task.py
index 09666c3..310c13b 100644
--- a/mujoco_maze/maze_task.py
+++ b/mujoco_maze/maze_task.py
@@ -329,7 +329,7 @@ class SubGoalTRoom(GoalRewardTRoom):
         )
 
 
-class NoRewardRoom(MazeTask):
+class NoRewardCorridor(MazeTask):
     REWARD_THRESHOLD: float = 0.0
     MAZE_SIZE_SCALING: Scaling = Scaling(4.0, 4.0, 1.0)
 
@@ -352,6 +352,25 @@ class NoRewardRoom(MazeTask):
         ]
 
 
+class GoalRewardCorridor(NoRewardCorridor):
+    REWARD_THRESHOLD: float = 0.9
+    PENALTY: float = -0.0001
+
+    def __init__(self, scale: float, goal: Tuple[float, float] = (3.0, -3.0)) -> None:
+        super().__init__(scale)
+        self.goals.append(MazeGoal(np.array(goal) * scale))
+
+    def reward(self, obs: np.ndarray) -> float:
+        for goal in self.goals:
+            if goal.neighbor(obs):
+                return goal.reward_scale
+        return self.PENALTY
+
+
+class DistRewardCorridor(GoalRewardCorridor, DistRewardMixIn):
+    pass
+
+
 class GoalRewardBlockMaze(GoalRewardUMaze):
     MAZE_SIZE_SCALING: Scaling = Scaling(8.0, 4.0, None)
     OBSERVE_BLOCKS: bool = True
@@ -483,7 +502,7 @@ class TaskRegistry:
         "4Rooms": [DistReward4Rooms, GoalReward4Rooms, SubGoal4Rooms],
         "TRoom": [DistRewardTRoom, GoalRewardTRoom, SubGoalTRoom],
         "BlockMaze": [DistRewardBlockMaze, GoalRewardBlockMaze],
-        "NoRewardRoom": [NoRewardRoom],
+        "Corridor": [DistRewardCorridor, GoalRewardCorridor, NoRewardCorridor],
         "Billiard": [
             DistRewardBilliard,
             GoalRewardBilliard,

From fe7e4d5a11d47c0f9398ad6b055ce0f46ed85b4d Mon Sep 17 00:00:00 2001
From: kngwyu <yuji.kanagawa@oist.jp>
Date: Tue, 18 May 2021 15:44:34 +0900
Subject: [PATCH 15/24] Now we can save video by right-click

---
 mujoco_maze/websock_viewer.py | 35 ++++++++++++++++++++++++++++++-----
 1 file changed, 30 insertions(+), 5 deletions(-)

diff --git a/mujoco_maze/websock_viewer.py b/mujoco_maze/websock_viewer.py
index a355b3f..647d5e1 100644
--- a/mujoco_maze/websock_viewer.py
+++ b/mujoco_maze/websock_viewer.py
@@ -6,6 +6,7 @@ import fastapi
 import uvicorn
 from PIL import Image
 
+
 HTML = """
 <!DOCTYPE html>
 <html>
@@ -13,10 +14,11 @@ HTML = """
     <title>MuJoCo maze visualizer</title>
   </head>
   <body>
+    <h2>MuJoCo Maze Visualizer</h2>
     <script>
-      var web_socket = new WebSocket('ws://127.0.0.1:{{port}}/ws');
-      web_socket.binaryType = "arraybuffer";
-      web_socket.onmessage = function(event) {
+      var ws_image = new WebSocket('ws://127.0.0.1:{{port}}/ws');
+      ws_image.binaryType = "arraybuffer";
+      ws_image.onmessage = function(event) {
           var canvas = document.getElementById('canvas');
           var ctx = canvas.getContext('2d');
           var blob = new Blob([event.data], {type:'image/png'});
@@ -28,10 +30,20 @@ HTML = """
           console.log(url);
           image.src = url;
       }
+      function saveVideo() {
+          var xhr = new XMLHttpRequest();
+          xhr.open('GET', 'https://via.placeholder.com/150', true);
+          xhr.responseType = 'blob';
+          xhr.onerror = err => {
+              alert('Video is not ready');
+          };
+          xhr.send();
+       }
     </script>
     <div>
       <canvas id="canvas" width="600" height="480"></canvas>
     </div>
+    <a href="video">Video</a>
   </body>
 </html>
 """
@@ -42,6 +54,7 @@ class _ServerWorker(mp.Process):
         super().__init__()
         self.pipe = pipe
         self.port = port
+        self.video_frames = []
 
     def _run_server(self) -> None:
 
@@ -49,19 +62,20 @@ class _ServerWorker(mp.Process):
         html = HTML.replace("{{port}}", str(self.port))
 
         @app.get("/")
-        async def get():
+        async def root():
             return fastapi.responses.HTMLResponse(html)
 
         server = None
 
         @app.websocket("/ws")
-        async def ws_send_image(websocket: fastapi.WebSocket):
+        async def ws(websocket: fastapi.WebSocket):
             await websocket.accept()
             loop = asyncio.get_running_loop()
             while True:
                 image_array = await loop.run_in_executor(None, self.pipe.recv)
                 if image_array is None:
                     break
+                self.video_frames.append(image_array)
                 image = Image.fromarray(image_array)
                 with io.BytesIO() as stream:
                     image.save(stream, format="png")
@@ -70,6 +84,17 @@ class _ServerWorker(mp.Process):
             await websocket.close()
             server.should_exit = True
 
+        @app.get("/video")
+        async def video():
+            import imageio
+
+            writer = imageio.get_writer("/tmp/mujoco-maze-video.mp4")
+            for frame in self.video_frames:
+                writer.append_data(frame)
+            writer.close()
+            video = open("/tmp/mujoco-maze-video.mp4", mode="rb")
+            return fastapi.responses.StreamingResponse(video, media_type="video/mp4")
+
         config = uvicorn.Config(app, port=self.port)
         server = uvicorn.Server(config)
         server.run()

From f2c5ff9f3cbc3e5f5d485220c2897306277ab2bd Mon Sep 17 00:00:00 2001
From: kngwyu <yuji.kanagawa@oist.jp>
Date: Tue, 18 May 2021 15:46:59 +0900
Subject: [PATCH 16/24] Add NoRewardBilliard

---
 mujoco_maze/maze_task.py | 19 ++++++++++++++-----
 1 file changed, 14 insertions(+), 5 deletions(-)

diff --git a/mujoco_maze/maze_task.py b/mujoco_maze/maze_task.py
index 310c13b..d8b4572 100644
--- a/mujoco_maze/maze_task.py
+++ b/mujoco_maze/maze_task.py
@@ -333,7 +333,7 @@ class NoRewardCorridor(MazeTask):
     REWARD_THRESHOLD: float = 0.0
     MAZE_SIZE_SCALING: Scaling = Scaling(4.0, 4.0, 1.0)
 
-    def reward(self, obs: np.ndarray) -> float:
+    def reward(self, _obs: np.ndarray) -> float:
         return 0.0
 
     @staticmethod
@@ -448,6 +448,14 @@ class DistRewardBilliard(GoalRewardBilliard):
         return -self.goals[0].euc_dist(obs[3:6]) / self.scale
 
 
+class NoRewardBilliard(GoalRewardBilliard):
+    def __init__(self, scale: float) -> None:
+        MazeTask.__init__(self, scale)
+
+    def reward(self, _obs: np.ndarray) -> float:
+        return 0.0
+
+
 class SubGoalBilliard(GoalRewardBilliard):
     def __init__(
         self,
@@ -504,10 +512,11 @@ class TaskRegistry:
         "BlockMaze": [DistRewardBlockMaze, GoalRewardBlockMaze],
         "Corridor": [DistRewardCorridor, GoalRewardCorridor, NoRewardCorridor],
         "Billiard": [
-            DistRewardBilliard,
-            GoalRewardBilliard,
-            SubGoalBilliard,
-            BanditBilliard,
+            DistRewardBilliard,  # v0
+            GoalRewardBilliard,  # v1
+            SubGoalBilliard,  # v2
+            BanditBilliard,  # v3
+            NoRewardBilliard,  # v4
         ],
     }
 

From 56df7df0a6fc86d108d1092225aac00bbbc02e72 Mon Sep 17 00:00:00 2001
From: kngwyu <yuji.kanagawa@oist.jp>
Date: Sat, 22 May 2021 13:10:08 +0900
Subject: [PATCH 17/24] chmod -x

---
 mujoco_maze/assets/ant.xml   | 0
 mujoco_maze/assets/point.xml | 0
 2 files changed, 0 insertions(+), 0 deletions(-)
 mode change 100755 => 100644 mujoco_maze/assets/ant.xml
 mode change 100755 => 100644 mujoco_maze/assets/point.xml

diff --git a/mujoco_maze/assets/ant.xml b/mujoco_maze/assets/ant.xml
old mode 100755
new mode 100644
diff --git a/mujoco_maze/assets/point.xml b/mujoco_maze/assets/point.xml
old mode 100755
new mode 100644

From 362bba2f79fa5a93095f5f8f50ead04342b4d6b6 Mon Sep 17 00:00:00 2001
From: kngwyu <yuji.kanagawa@oist.jp>
Date: Thu, 27 May 2021 00:05:09 +0900
Subject: [PATCH 18/24] SquareRoom

---
 mujoco_maze/maze_task.py | 32 ++++++++++++++++++++++++++++++++
 1 file changed, 32 insertions(+)

diff --git a/mujoco_maze/maze_task.py b/mujoco_maze/maze_task.py
index d8b4572..4ee801b 100644
--- a/mujoco_maze/maze_task.py
+++ b/mujoco_maze/maze_task.py
@@ -145,6 +145,37 @@ class DistRewardSimpleRoom(GoalRewardSimpleRoom, DistRewardMixIn):
     pass
 
 
+class GoalRewardSquareRoom(GoalRewardUMaze):
+    MAZE_SIZE_SCALING: Scaling = Scaling(4.0, 4.0, 2.0)
+
+    def __init__(self, scale: float, goal: Tuple[float, float] = (2.0, 0.0)) -> None:
+        super().__init__(scale)
+        self.goals = [MazeGoal(np.array(goal) * scale)]
+
+    @staticmethod
+    def create_maze() -> List[List[MazeCell]]:
+        E, B, R = MazeCell.EMPTY, MazeCell.BLOCK, MazeCell.ROBOT
+        return [
+            [B, B, B, B, B],
+            [B, E, E, E, B],
+            [B, R, E, E, B],
+            [B, E, E, E, B],
+            [B, B, B, B, B],
+        ]
+
+
+class NoRewardSquareRoom(GoalRewardSimpleRoom):
+    def __init__(self, scale: float) -> None:
+        super().__init__(scale)
+
+    def reward(self) -> float:
+        return 0.0
+
+
+class DistRewardSquareRoom(GoalRewardSquareRoom, DistRewardMixIn):
+    pass
+
+
 class GoalRewardPush(GoalRewardUMaze):
     OBSERVE_BLOCKS: bool = True
 
@@ -503,6 +534,7 @@ class BanditBilliard(SubGoalBilliard):
 class TaskRegistry:
     REGISTRY: Dict[str, List[Type[MazeTask]]] = {
         "SimpleRoom": [DistRewardSimpleRoom, GoalRewardSimpleRoom],
+        "SquareRoom": [DistRewardSquareRoom, GoalRewardSquareRoom, NoRewardSquareRoom],
         "UMaze": [DistRewardUMaze, GoalRewardUMaze],
         "Push": [DistRewardPush, GoalRewardPush],
         "Fall": [DistRewardFall, GoalRewardFall],

From 3812c034c9cf084b8e45d41ce6938613ec22fb2a Mon Sep 17 00:00:00 2001
From: kngwyu <yuji.kanagawa@oist.jp>
Date: Thu, 27 May 2021 01:06:06 +0900
Subject: [PATCH 19/24] Fix NoRewardSquareRoom.reward

---
 mujoco_maze/maze_task.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mujoco_maze/maze_task.py b/mujoco_maze/maze_task.py
index 4ee801b..9e102a9 100644
--- a/mujoco_maze/maze_task.py
+++ b/mujoco_maze/maze_task.py
@@ -168,7 +168,7 @@ class NoRewardSquareRoom(GoalRewardSimpleRoom):
     def __init__(self, scale: float) -> None:
         super().__init__(scale)
 
-    def reward(self) -> float:
+    def reward(self, _obs: np.ndarray) -> float:
         return 0.0
 
 

From ddb8a5f3c0b66d6a1121dc1c96cef85f23a6e47a Mon Sep 17 00:00:00 2001
From: kngwyu <yuji.kanagawa@oist.jp>
Date: Fri, 28 May 2021 11:44:11 +0900
Subject: [PATCH 20/24] Change the start position of SquareRoom

---
 mujoco_maze/maze_task.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/mujoco_maze/maze_task.py b/mujoco_maze/maze_task.py
index 9e102a9..21b4ce6 100644
--- a/mujoco_maze/maze_task.py
+++ b/mujoco_maze/maze_task.py
@@ -148,7 +148,7 @@ class DistRewardSimpleRoom(GoalRewardSimpleRoom, DistRewardMixIn):
 class GoalRewardSquareRoom(GoalRewardUMaze):
     MAZE_SIZE_SCALING: Scaling = Scaling(4.0, 4.0, 2.0)
 
-    def __init__(self, scale: float, goal: Tuple[float, float] = (2.0, 0.0)) -> None:
+    def __init__(self, scale: float, goal: Tuple[float, float] = (1.0, 0.0)) -> None:
         super().__init__(scale)
         self.goals = [MazeGoal(np.array(goal) * scale)]
 
@@ -158,7 +158,7 @@ class GoalRewardSquareRoom(GoalRewardUMaze):
         return [
             [B, B, B, B, B],
             [B, E, E, E, B],
-            [B, R, E, E, B],
+            [B, E, R, E, B],
             [B, E, E, E, B],
             [B, B, B, B, B],
         ]

From 1af434cd1672e8030c4d1a918350d7e075d51e2c Mon Sep 17 00:00:00 2001
From: kngwyu <yuji.kanagawa@oist.jp>
Date: Wed, 2 Jun 2021 10:24:58 +0900
Subject: [PATCH 21/24] [Actions] apt update

---
 .github/workflows/main.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index a34055f..4e33292 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -26,7 +26,7 @@ jobs:
       - name: Install Poetry
         run: pip install poetry
       - name: Install Mujoco Dependencies
-        run: sudo apt install libosmesa6-dev libglfw3 libglew-dev patchelf libgl1-mesa-glx -y
+        run: sudo apt update && sudo apt install libosmesa6-dev libglfw3 libglew-dev patchelf libgl1-mesa-glx -y
       - name: Install Mujoco
         run: |
           mkdir ~/.mujoco

From 46a7d74a1b850c48284f39e3a459d035c56e449a Mon Sep 17 00:00:00 2001
From: kngwyu <yuji.kngw.80s.revive@gmail.com>
Date: Thu, 3 Jun 2021 17:22:24 +0900
Subject: [PATCH 22/24] Require Python 3.7

---
 .github/workflows/main.yml |   2 +-
 poetry.lock                | 590 +++++++++++++++++++------------------
 pyproject.toml             |   2 +-
 3 files changed, 300 insertions(+), 294 deletions(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 4e33292..d8887a8 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -14,7 +14,7 @@ jobs:
   ci:
     strategy:
       matrix:
-        python-version: [3.6, 3.7, 3.8]
+        python-version: [3.7, 3.8]
     runs-on: ubuntu-18.04
     env:
       LD_LIBRARY_PATH: /home/runner/.mujoco/mujoco200/bin
diff --git a/poetry.lock b/poetry.lock
index 5faf0de..832a57f 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -1,117 +1,109 @@
 [[package]]
-category = "dev"
-description = "Atomic file writes."
 name = "atomicwrites"
+version = "1.4.0"
+description = "Atomic file writes."
+category = "dev"
 optional = false
 python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
-version = "1.4.0"
 
 [[package]]
-category = "dev"
-description = "Classes Without Boilerplate"
 name = "attrs"
+version = "21.2.0"
+description = "Classes Without Boilerplate"
+category = "dev"
 optional = false
-python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
-version = "20.2.0"
+python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
 
 [package.extras]
-dev = ["coverage (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface", "sphinx", "sphinx-rtd-theme", "pre-commit"]
-docs = ["sphinx", "sphinx-rtd-theme", "zope.interface"]
-tests = ["coverage (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface"]
-tests_no_zope = ["coverage (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six"]
+dev = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "zope.interface", "furo", "sphinx", "sphinx-notfound-page", "pre-commit"]
+docs = ["furo", "sphinx", "zope.interface", "sphinx-notfound-page"]
+tests = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "zope.interface"]
+tests_no_zope = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins"]
 
 [[package]]
-category = "main"
-description = "Foreign Function Interface for Python calling C code."
 name = "cffi"
+version = "1.14.5"
+description = "Foreign Function Interface for Python calling C code."
+category = "main"
 optional = false
 python-versions = "*"
-version = "1.14.3"
 
 [package.dependencies]
 pycparser = "*"
 
 [[package]]
-category = "main"
-description = "Extended pickling support for Python objects"
 name = "cloudpickle"
+version = "1.6.0"
+description = "Extended pickling support for Python objects"
+category = "main"
 optional = false
 python-versions = ">=3.5"
-version = "1.6.0"
 
 [[package]]
-category = "dev"
-description = "Cross-platform colored terminal text."
-marker = "sys_platform == \"win32\""
 name = "colorama"
+version = "0.4.4"
+description = "Cross-platform colored terminal text."
+category = "dev"
 optional = false
 python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
-version = "0.4.3"
 
 [[package]]
-category = "main"
-description = "The Cython compiler for writing C extensions for the Python language."
 name = "cython"
+version = "0.29.23"
+description = "The Cython compiler for writing C extensions for the Python language."
+category = "main"
 optional = false
 python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*"
-version = "0.29.21"
 
 [[package]]
-category = "main"
-description = "A python package that provides useful locks."
 name = "fasteners"
+version = "0.16"
+description = "A python package that provides useful locks."
+category = "main"
 optional = false
 python-versions = "*"
-version = "0.15"
 
 [package.dependencies]
-monotonic = ">=0.1"
 six = "*"
 
 [[package]]
-category = "main"
-description = "Clean single-source support for Python 3 and 2"
-name = "future"
-optional = false
-python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*"
-version = "0.18.2"
-
-[[package]]
-category = "main"
-description = "A ctypes-based wrapper for GLFW3."
 name = "glfw"
+version = "2.1.0"
+description = "A ctypes-based wrapper for GLFW3."
+category = "main"
 optional = false
 python-versions = "*"
-version = "2.0.0"
 
 [[package]]
-category = "main"
-description = "The OpenAI Gym: A toolkit for developing and comparing your reinforcement learning agents."
 name = "gym"
+version = "0.18.3"
+description = "The OpenAI Gym: A toolkit for developing and comparing your reinforcement learning agents."
+category = "main"
 optional = false
-python-versions = ">=3.5"
-version = "0.17.3"
+python-versions = ">=3.6"
 
 [package.dependencies]
 cloudpickle = ">=1.2.0,<1.7.0"
 numpy = ">=1.10.4"
-pyglet = ">=1.4.0,<=1.5.0"
+Pillow = "<=8.2.0"
+pyglet = ">=1.4.0,<=1.5.15"
 scipy = "*"
 
 [package.extras]
-all = ["atari_py (>=0.2.0,<0.3.0)", "pillow", "opencv-python", "box2d-py (>=2.3.5,<2.4.0)", "mujoco_py (>=1.50,<2.0)", "imageio", "mujoco_py (>=1.50,<2.0)", "imageio"]
-atari = ["atari_py (>=0.2.0,<0.3.0)", "pillow", "opencv-python"]
+all = ["box2d-py (>=2.3.5,<2.4.0)", "opencv-python (>=3)", "imageio", "atari_py (>=0.2.0,<0.3.0)", "mujoco_py (>=1.50,<2.0)"]
+atari = ["atari_py (>=0.2.0,<0.3.0)", "opencv-python (>=3)"]
 box2d = ["box2d-py (>=2.3.5,<2.4.0)"]
 mujoco = ["mujoco_py (>=1.50,<2.0)", "imageio"]
+nomujoco = ["box2d-py (>=2.3.5,<2.4.0)", "opencv-python (>=3)", "atari_py (>=0.2.0,<0.3.0)"]
 robotics = ["mujoco_py (>=1.50,<2.0)", "imageio"]
 
 [[package]]
-category = "main"
-description = "Library for reading and writing a wide range of image, video, scientific, and volumetric data formats."
 name = "imageio"
+version = "2.9.0"
+description = "Library for reading and writing a wide range of image, video, scientific, and volumetric data formats."
+category = "main"
 optional = false
 python-versions = ">=3.5"
-version = "2.9.0"
 
 [package.dependencies]
 numpy = "*"
@@ -125,165 +117,159 @@ gdal = ["gdal"]
 itk = ["itk"]
 
 [[package]]
-category = "dev"
-description = "Read metadata from Python packages"
-marker = "python_version < \"3.8\""
 name = "importlib-metadata"
+version = "4.4.0"
+description = "Read metadata from Python packages"
+category = "dev"
 optional = false
-python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7"
-version = "2.0.0"
+python-versions = ">=3.6"
 
 [package.dependencies]
+typing-extensions = {version = ">=3.6.4", markers = "python_version < \"3.8\""}
 zipp = ">=0.5"
 
 [package.extras]
-docs = ["sphinx", "rst.linker"]
-testing = ["packaging", "pep517", "importlib-resources (>=1.3)"]
+docs = ["sphinx", "jaraco.packaging (>=8.2)", "rst.linker (>=1.9)"]
+testing = ["pytest (>=4.6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.0.1)", "packaging", "pep517", "pyfakefs", "flufl.flake8", "pytest-black (>=0.3.7)", "pytest-mypy", "importlib-resources (>=1.3)"]
 
 [[package]]
-category = "main"
-description = "An implementation of time.monotonic() for Python 2 & < 3.3"
-name = "monotonic"
-optional = false
-python-versions = "*"
-version = "1.5"
-
-[[package]]
-category = "dev"
-description = "More routines for operating on iterables, beyond itertools"
 name = "more-itertools"
+version = "8.8.0"
+description = "More routines for operating on iterables, beyond itertools"
+category = "dev"
 optional = false
 python-versions = ">=3.5"
-version = "8.5.0"
 
 [[package]]
-category = "main"
-description = ""
 name = "mujoco-py"
+version = "2.0.2.13"
+description = ""
+category = "main"
 optional = false
 python-versions = ">=3.6"
-version = "2.0.2.13"
 
 [package.dependencies]
-Cython = ">=0.27.2"
 cffi = ">=1.10"
+Cython = ">=0.27.2"
 fasteners = ">=0.15,<1.0"
 glfw = ">=1.4.0"
 imageio = ">=2.1.2"
 numpy = ">=1.11"
 
 [[package]]
-category = "main"
-description = "NumPy is the fundamental package for array computing with Python."
 name = "numpy"
+version = "1.20.3"
+description = "NumPy is the fundamental package for array computing with Python."
+category = "main"
+optional = false
+python-versions = ">=3.7"
+
+[[package]]
+name = "pillow"
+version = "8.2.0"
+description = "Python Imaging Library (Fork)"
+category = "main"
 optional = false
 python-versions = ">=3.6"
-version = "1.19.2"
 
 [[package]]
-category = "main"
-description = "Python Imaging Library (Fork)"
-name = "pillow"
-optional = false
-python-versions = ">=3.5"
-version = "7.2.0"
-
-[[package]]
-category = "dev"
-description = "plugin and hook calling mechanisms for python"
 name = "pluggy"
+version = "0.13.1"
+description = "plugin and hook calling mechanisms for python"
+category = "dev"
 optional = false
 python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
-version = "0.13.1"
 
 [package.dependencies]
-[package.dependencies.importlib-metadata]
-python = "<3.8"
-version = ">=0.12"
+importlib-metadata = {version = ">=0.12", markers = "python_version < \"3.8\""}
 
 [package.extras]
 dev = ["pre-commit", "tox"]
 
 [[package]]
-category = "dev"
-description = "library with cross-python path, ini-parsing, io, code, log facilities"
 name = "py"
+version = "1.10.0"
+description = "library with cross-python path, ini-parsing, io, code, log facilities"
+category = "dev"
 optional = false
 python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
-version = "1.9.0"
 
 [[package]]
-category = "main"
-description = "C parser in Python"
 name = "pycparser"
+version = "2.20"
+description = "C parser in Python"
+category = "main"
 optional = false
 python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
-version = "2.20"
 
 [[package]]
-category = "main"
-description = "Cross-platform windowing and multimedia library"
 name = "pyglet"
+version = "1.5.15"
+description = "Cross-platform windowing and multimedia library"
+category = "main"
 optional = false
 python-versions = "*"
-version = "1.5.0"
-
-[package.dependencies]
-future = "*"
 
 [[package]]
-category = "dev"
-description = "pytest: simple powerful testing with Python"
 name = "pytest"
+version = "3.10.1"
+description = "pytest: simple powerful testing with Python"
+category = "dev"
 optional = false
 python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
-version = "3.10.1"
 
 [package.dependencies]
 atomicwrites = ">=1.0"
 attrs = ">=17.4.0"
-colorama = "*"
+colorama = {version = "*", markers = "sys_platform == \"win32\""}
 more-itertools = ">=4.0.0"
 pluggy = ">=0.7"
 py = ">=1.5.0"
-setuptools = "*"
 six = ">=1.10.0"
 
 [[package]]
-category = "main"
-description = "SciPy: Scientific Library for Python"
 name = "scipy"
+version = "1.6.1"
+description = "SciPy: Scientific Library for Python"
+category = "main"
 optional = false
-python-versions = ">=3.6"
-version = "1.5.2"
+python-versions = ">=3.7"
 
 [package.dependencies]
-numpy = ">=1.14.5"
+numpy = ">=1.16.5"
 
 [[package]]
-category = "main"
-description = "Python 2 and 3 compatibility utilities"
 name = "six"
+version = "1.16.0"
+description = "Python 2 and 3 compatibility utilities"
+category = "main"
 optional = false
 python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*"
-version = "1.15.0"
 
 [[package]]
+name = "typing-extensions"
+version = "3.10.0.0"
+description = "Backported and Experimental Type Hints for Python 3.5+"
 category = "dev"
-description = "Backport of pathlib-compatible object wrapper for zip files"
-marker = "python_version < \"3.8\""
+optional = false
+python-versions = "*"
+
+[[package]]
 name = "zipp"
+version = "3.4.1"
+description = "Backport of pathlib-compatible object wrapper for zip files"
+category = "dev"
 optional = false
 python-versions = ">=3.6"
-version = "3.3.0"
 
 [package.extras]
-docs = ["sphinx", "jaraco.packaging (>=3.2)", "rst.linker (>=1.9)"]
-testing = ["pytest (>=3.5,<3.7.3 || >3.7.3)", "pytest-checkdocs (>=1.2.3)", "pytest-flake8", "pytest-cov", "jaraco.test (>=3.2.0)", "jaraco.itertools", "func-timeout", "pytest-black (>=0.3.7)", "pytest-mypy"]
+docs = ["sphinx", "jaraco.packaging (>=8.2)", "rst.linker (>=1.9)"]
+testing = ["pytest (>=4.6)", "pytest-checkdocs (>=1.2.3)", "pytest-flake8", "pytest-cov", "pytest-enabler", "jaraco.itertools", "func-timeout", "pytest-black (>=0.3.7)", "pytest-mypy"]
 
 [metadata]
-content-hash = "7ae1ca698619b46069b373d39e9b71110d4d71c3d9b4b97b7e22f6a80abe47f6"
-python-versions = "^3.6"  # Compatible python versions must be declared here
+lock-version = "1.1"
+python-versions = ">=3.7"  # Compatible python versions must be declared here
+content-hash = "9234f9b51f7c3fab1f9eefcf7f08e01355ce357941e935359e3145e95fb676f3"
 
 [metadata.files]
 atomicwrites = [
@@ -291,229 +277,249 @@ atomicwrites = [
     {file = "atomicwrites-1.4.0.tar.gz", hash = "sha256:ae70396ad1a434f9c7046fd2dd196fc04b12f9e91ffb859164193be8b6168a7a"},
 ]
 attrs = [
-    {file = "attrs-20.2.0-py2.py3-none-any.whl", hash = "sha256:fce7fc47dfc976152e82d53ff92fa0407700c21acd20886a13777a0d20e655dc"},
-    {file = "attrs-20.2.0.tar.gz", hash = "sha256:26b54ddbbb9ee1d34d5d3668dd37d6cf74990ab23c828c2888dccdceee395594"},
+    {file = "attrs-21.2.0-py2.py3-none-any.whl", hash = "sha256:149e90d6d8ac20db7a955ad60cf0e6881a3f20d37096140088356da6c716b0b1"},
+    {file = "attrs-21.2.0.tar.gz", hash = "sha256:ef6aaac3ca6cd92904cdd0d83f629a15f18053ec84e6432106f7a4d04ae4f5fb"},
 ]
 cffi = [
-    {file = "cffi-1.14.3-2-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:3eeeb0405fd145e714f7633a5173318bd88d8bbfc3dd0a5751f8c4f70ae629bc"},
-    {file = "cffi-1.14.3-2-cp35-cp35m-macosx_10_9_x86_64.whl", hash = "sha256:cb763ceceae04803adcc4e2d80d611ef201c73da32d8f2722e9d0ab0c7f10768"},
-    {file = "cffi-1.14.3-2-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:44f60519595eaca110f248e5017363d751b12782a6f2bd6a7041cba275215f5d"},
-    {file = "cffi-1.14.3-2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c53af463f4a40de78c58b8b2710ade243c81cbca641e34debf3396a9640d6ec1"},
-    {file = "cffi-1.14.3-2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:33c6cdc071ba5cd6d96769c8969a0531be2d08c2628a0143a10a7dcffa9719ca"},
-    {file = "cffi-1.14.3-2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c11579638288e53fc94ad60022ff1b67865363e730ee41ad5e6f0a17188b327a"},
-    {file = "cffi-1.14.3-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:3cb3e1b9ec43256c4e0f8d2837267a70b0e1ca8c4f456685508ae6106b1f504c"},
-    {file = "cffi-1.14.3-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:f0620511387790860b249b9241c2f13c3a80e21a73e0b861a2df24e9d6f56730"},
-    {file = "cffi-1.14.3-cp27-cp27m-win32.whl", hash = "sha256:005f2bfe11b6745d726dbb07ace4d53f057de66e336ff92d61b8c7e9c8f4777d"},
-    {file = "cffi-1.14.3-cp27-cp27m-win_amd64.whl", hash = "sha256:2f9674623ca39c9ebe38afa3da402e9326c245f0f5ceff0623dccdac15023e05"},
-    {file = "cffi-1.14.3-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:09e96138280241bd355cd585148dec04dbbedb4f46128f340d696eaafc82dd7b"},
-    {file = "cffi-1.14.3-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:3363e77a6176afb8823b6e06db78c46dbc4c7813b00a41300a4873b6ba63b171"},
-    {file = "cffi-1.14.3-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:0ef488305fdce2580c8b2708f22d7785ae222d9825d3094ab073e22e93dfe51f"},
-    {file = "cffi-1.14.3-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:0b1ad452cc824665ddc682400b62c9e4f5b64736a2ba99110712fdee5f2505c4"},
-    {file = "cffi-1.14.3-cp35-cp35m-win32.whl", hash = "sha256:85ba797e1de5b48aa5a8427b6ba62cf69607c18c5d4eb747604b7302f1ec382d"},
-    {file = "cffi-1.14.3-cp35-cp35m-win_amd64.whl", hash = "sha256:e66399cf0fc07de4dce4f588fc25bfe84a6d1285cc544e67987d22663393926d"},
-    {file = "cffi-1.14.3-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:15f351bed09897fbda218e4db5a3d5c06328862f6198d4fb385f3e14e19decb3"},
-    {file = "cffi-1.14.3-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:4d7c26bfc1ea9f92084a1d75e11999e97b62d63128bcc90c3624d07813c52808"},
-    {file = "cffi-1.14.3-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:23e5d2040367322824605bc29ae8ee9175200b92cb5483ac7d466927a9b3d537"},
-    {file = "cffi-1.14.3-cp36-cp36m-win32.whl", hash = "sha256:a624fae282e81ad2e4871bdb767e2c914d0539708c0f078b5b355258293c98b0"},
-    {file = "cffi-1.14.3-cp36-cp36m-win_amd64.whl", hash = "sha256:de31b5164d44ef4943db155b3e8e17929707cac1e5bd2f363e67a56e3af4af6e"},
-    {file = "cffi-1.14.3-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:f92cdecb618e5fa4658aeb97d5eb3d2f47aa94ac6477c6daf0f306c5a3b9e6b1"},
-    {file = "cffi-1.14.3-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:22399ff4870fb4c7ef19fff6eeb20a8bbf15571913c181c78cb361024d574579"},
-    {file = "cffi-1.14.3-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:f4eae045e6ab2bb54ca279733fe4eb85f1effda392666308250714e01907f394"},
-    {file = "cffi-1.14.3-cp37-cp37m-win32.whl", hash = "sha256:b0358e6fefc74a16f745afa366acc89f979040e0cbc4eec55ab26ad1f6a9bfbc"},
-    {file = "cffi-1.14.3-cp37-cp37m-win_amd64.whl", hash = "sha256:6642f15ad963b5092d65aed022d033c77763515fdc07095208f15d3563003869"},
-    {file = "cffi-1.14.3-cp38-cp38-manylinux1_i686.whl", hash = "sha256:2791f68edc5749024b4722500e86303a10d342527e1e3bcac47f35fbd25b764e"},
-    {file = "cffi-1.14.3-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:529c4ed2e10437c205f38f3691a68be66c39197d01062618c55f74294a4a4828"},
-    {file = "cffi-1.14.3-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:8f0f1e499e4000c4c347a124fa6a27d37608ced4fe9f7d45070563b7c4c370c9"},
-    {file = "cffi-1.14.3-cp38-cp38-win32.whl", hash = "sha256:3b8eaf915ddc0709779889c472e553f0d3e8b7bdf62dab764c8921b09bf94522"},
-    {file = "cffi-1.14.3-cp38-cp38-win_amd64.whl", hash = "sha256:bbd2f4dfee1079f76943767fce837ade3087b578aeb9f69aec7857d5bf25db15"},
-    {file = "cffi-1.14.3-cp39-cp39-manylinux1_i686.whl", hash = "sha256:cc75f58cdaf043fe6a7a6c04b3b5a0e694c6a9e24050967747251fb80d7bce0d"},
-    {file = "cffi-1.14.3-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:bf39a9e19ce7298f1bd6a9758fa99707e9e5b1ebe5e90f2c3913a47bc548747c"},
-    {file = "cffi-1.14.3-cp39-cp39-win32.whl", hash = "sha256:d80998ed59176e8cba74028762fbd9b9153b9afc71ea118e63bbf5d4d0f9552b"},
-    {file = "cffi-1.14.3-cp39-cp39-win_amd64.whl", hash = "sha256:c150eaa3dadbb2b5339675b88d4573c1be3cb6f2c33a6c83387e10cc0bf05bd3"},
-    {file = "cffi-1.14.3.tar.gz", hash = "sha256:f92f789e4f9241cd262ad7a555ca2c648a98178a953af117ef7fad46aa1d5591"},
+    {file = "cffi-1.14.5-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:bb89f306e5da99f4d922728ddcd6f7fcebb3241fc40edebcb7284d7514741991"},
+    {file = "cffi-1.14.5-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:34eff4b97f3d982fb93e2831e6750127d1355a923ebaeeb565407b3d2f8d41a1"},
+    {file = "cffi-1.14.5-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:99cd03ae7988a93dd00bcd9d0b75e1f6c426063d6f03d2f90b89e29b25b82dfa"},
+    {file = "cffi-1.14.5-cp27-cp27m-win32.whl", hash = "sha256:65fa59693c62cf06e45ddbb822165394a288edce9e276647f0046e1ec26920f3"},
+    {file = "cffi-1.14.5-cp27-cp27m-win_amd64.whl", hash = "sha256:51182f8927c5af975fece87b1b369f722c570fe169f9880764b1ee3bca8347b5"},
+    {file = "cffi-1.14.5-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:43e0b9d9e2c9e5d152946b9c5fe062c151614b262fda2e7b201204de0b99e482"},
+    {file = "cffi-1.14.5-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:cbde590d4faaa07c72bf979734738f328d239913ba3e043b1e98fe9a39f8b2b6"},
+    {file = "cffi-1.14.5-cp35-cp35m-macosx_10_9_x86_64.whl", hash = "sha256:5de7970188bb46b7bf9858eb6890aad302577a5f6f75091fd7cdd3ef13ef3045"},
+    {file = "cffi-1.14.5-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:a465da611f6fa124963b91bf432d960a555563efe4ed1cc403ba5077b15370aa"},
+    {file = "cffi-1.14.5-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:d42b11d692e11b6634f7613ad8df5d6d5f8875f5d48939520d351007b3c13406"},
+    {file = "cffi-1.14.5-cp35-cp35m-win32.whl", hash = "sha256:72d8d3ef52c208ee1c7b2e341f7d71c6fd3157138abf1a95166e6165dd5d4369"},
+    {file = "cffi-1.14.5-cp35-cp35m-win_amd64.whl", hash = "sha256:29314480e958fd8aab22e4a58b355b629c59bf5f2ac2492b61e3dc06d8c7a315"},
+    {file = "cffi-1.14.5-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:3d3dd4c9e559eb172ecf00a2a7517e97d1e96de2a5e610bd9b68cea3925b4892"},
+    {file = "cffi-1.14.5-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:48e1c69bbacfc3d932221851b39d49e81567a4d4aac3b21258d9c24578280058"},
+    {file = "cffi-1.14.5-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:69e395c24fc60aad6bb4fa7e583698ea6cc684648e1ffb7fe85e3c1ca131a7d5"},
+    {file = "cffi-1.14.5-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:9e93e79c2551ff263400e1e4be085a1210e12073a31c2011dbbda14bda0c6132"},
+    {file = "cffi-1.14.5-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:24ec4ff2c5c0c8f9c6b87d5bb53555bf267e1e6f70e52e5a9740d32861d36b6f"},
+    {file = "cffi-1.14.5-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3c3f39fa737542161d8b0d680df2ec249334cd70a8f420f71c9304bd83c3cbed"},
+    {file = "cffi-1.14.5-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:681d07b0d1e3c462dd15585ef5e33cb021321588bebd910124ef4f4fb71aef55"},
+    {file = "cffi-1.14.5-cp36-cp36m-win32.whl", hash = "sha256:58e3f59d583d413809d60779492342801d6e82fefb89c86a38e040c16883be53"},
+    {file = "cffi-1.14.5-cp36-cp36m-win_amd64.whl", hash = "sha256:005a36f41773e148deac64b08f233873a4d0c18b053d37da83f6af4d9087b813"},
+    {file = "cffi-1.14.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:2894f2df484ff56d717bead0a5c2abb6b9d2bf26d6960c4604d5c48bbc30ee73"},
+    {file = "cffi-1.14.5-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:0857f0ae312d855239a55c81ef453ee8fd24136eaba8e87a2eceba644c0d4c06"},
+    {file = "cffi-1.14.5-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:cd2868886d547469123fadc46eac7ea5253ea7fcb139f12e1dfc2bbd406427d1"},
+    {file = "cffi-1.14.5-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:35f27e6eb43380fa080dccf676dece30bef72e4a67617ffda586641cd4508d49"},
+    {file = "cffi-1.14.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:06d7cd1abac2ffd92e65c0609661866709b4b2d82dd15f611e602b9b188b0b69"},
+    {file = "cffi-1.14.5-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0f861a89e0043afec2a51fd177a567005847973be86f709bbb044d7f42fc4e05"},
+    {file = "cffi-1.14.5-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cc5a8e069b9ebfa22e26d0e6b97d6f9781302fe7f4f2b8776c3e1daea35f1adc"},
+    {file = "cffi-1.14.5-cp37-cp37m-win32.whl", hash = "sha256:9ff227395193126d82e60319a673a037d5de84633f11279e336f9c0f189ecc62"},
+    {file = "cffi-1.14.5-cp37-cp37m-win_amd64.whl", hash = "sha256:9cf8022fb8d07a97c178b02327b284521c7708d7c71a9c9c355c178ac4bbd3d4"},
+    {file = "cffi-1.14.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:8b198cec6c72df5289c05b05b8b0969819783f9418e0409865dac47288d2a053"},
+    {file = "cffi-1.14.5-cp38-cp38-manylinux1_i686.whl", hash = "sha256:ad17025d226ee5beec591b52800c11680fca3df50b8b29fe51d882576e039ee0"},
+    {file = "cffi-1.14.5-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:6c97d7350133666fbb5cf4abdc1178c812cb205dc6f41d174a7b0f18fb93337e"},
+    {file = "cffi-1.14.5-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:8ae6299f6c68de06f136f1f9e69458eae58f1dacf10af5c17353eae03aa0d827"},
+    {file = "cffi-1.14.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:04c468b622ed31d408fea2346bec5bbffba2cc44226302a0de1ade9f5ea3d373"},
+    {file = "cffi-1.14.5-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:06db6321b7a68b2bd6df96d08a5adadc1fa0e8f419226e25b2a5fbf6ccc7350f"},
+    {file = "cffi-1.14.5-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:293e7ea41280cb28c6fcaaa0b1aa1f533b8ce060b9e701d78511e1e6c4a1de76"},
+    {file = "cffi-1.14.5-cp38-cp38-win32.whl", hash = "sha256:b85eb46a81787c50650f2392b9b4ef23e1f126313b9e0e9013b35c15e4288e2e"},
+    {file = "cffi-1.14.5-cp38-cp38-win_amd64.whl", hash = "sha256:1f436816fc868b098b0d63b8920de7d208c90a67212546d02f84fe78a9c26396"},
+    {file = "cffi-1.14.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:1071534bbbf8cbb31b498d5d9db0f274f2f7a865adca4ae429e147ba40f73dea"},
+    {file = "cffi-1.14.5-cp39-cp39-manylinux1_i686.whl", hash = "sha256:9de2e279153a443c656f2defd67769e6d1e4163952b3c622dcea5b08a6405322"},
+    {file = "cffi-1.14.5-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:6e4714cc64f474e4d6e37cfff31a814b509a35cb17de4fb1999907575684479c"},
+    {file = "cffi-1.14.5-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:158d0d15119b4b7ff6b926536763dc0714313aa59e320ddf787502c70c4d4bee"},
+    {file = "cffi-1.14.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1bf1ac1984eaa7675ca8d5745a8cb87ef7abecb5592178406e55858d411eadc0"},
+    {file = "cffi-1.14.5-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:df5052c5d867c1ea0b311fb7c3cd28b19df469c056f7fdcfe88c7473aa63e333"},
+    {file = "cffi-1.14.5-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:24a570cd11895b60829e941f2613a4f79df1a27344cbbb82164ef2e0116f09c7"},
+    {file = "cffi-1.14.5-cp39-cp39-win32.whl", hash = "sha256:afb29c1ba2e5a3736f1c301d9d0abe3ec8b86957d04ddfa9d7a6a42b9367e396"},
+    {file = "cffi-1.14.5-cp39-cp39-win_amd64.whl", hash = "sha256:f2d45f97ab6bb54753eab54fffe75aaf3de4ff2341c9daee1987ee1837636f1d"},
+    {file = "cffi-1.14.5.tar.gz", hash = "sha256:fd78e5fee591709f32ef6edb9a015b4aa1a5022598e36227500c8f4e02328d9c"},
 ]
 cloudpickle = [
     {file = "cloudpickle-1.6.0-py3-none-any.whl", hash = "sha256:3a32d0eb0bc6f4d0c57fbc4f3e3780f7a81e6fee0fa935072884d58ae8e1cc7c"},
     {file = "cloudpickle-1.6.0.tar.gz", hash = "sha256:9bc994f9e9447593bd0a45371f0e7ac7333710fcf64a4eb9834bf149f4ef2f32"},
 ]
 colorama = [
-    {file = "colorama-0.4.3-py2.py3-none-any.whl", hash = "sha256:7d73d2a99753107a36ac6b455ee49046802e59d9d076ef8e47b61499fa29afff"},
-    {file = "colorama-0.4.3.tar.gz", hash = "sha256:e96da0d330793e2cb9485e9ddfd918d456036c7149416295932478192f4436a1"},
+    {file = "colorama-0.4.4-py2.py3-none-any.whl", hash = "sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2"},
+    {file = "colorama-0.4.4.tar.gz", hash = "sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b"},
 ]
 cython = [
-    {file = "Cython-0.29.21-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:c541b2b49c6638f2b5beb9316726db84a8d1c132bf31b942dae1f9c7f6ad3b92"},
-    {file = "Cython-0.29.21-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:b8d8497091c1dc8705d1575c71e908a93b1f127a174b2d472020f3d84263ac28"},
-    {file = "Cython-0.29.21-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:695a6bcaf9e12b1e471dfce96bbecf22a1487adc2ac6106b15960a2b51b97f5d"},
-    {file = "Cython-0.29.21-cp27-cp27m-win32.whl", hash = "sha256:171b9f70ceafcec5852089d0f9c1e75b0d554f46c882cd4e2e4acaba9bd7d148"},
-    {file = "Cython-0.29.21-cp27-cp27m-win_amd64.whl", hash = "sha256:539e59949aab4955c143a468810123bf22d3e8556421e1ce2531ed4893914ca0"},
-    {file = "Cython-0.29.21-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:e93acd1f603a0c1786e0841f066ae7cef014cf4750e3cd06fd03cfdf46361419"},
-    {file = "Cython-0.29.21-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:2922e3031ba9ebbe7cb9200b585cc33b71d66023d78450dcb883f824f4969371"},
-    {file = "Cython-0.29.21-cp34-cp34m-manylinux1_i686.whl", hash = "sha256:497841897942f734b0abc2dead2d4009795ee992267a70a23485fd0e937edc0b"},
-    {file = "Cython-0.29.21-cp34-cp34m-manylinux1_x86_64.whl", hash = "sha256:0ac10bf476476a9f7ef61ec6e44c280ef434473124ad31d3132b720f7b0e8d2a"},
-    {file = "Cython-0.29.21-cp34-cp34m-win32.whl", hash = "sha256:31c71a615f38401b0dc1f2a5a9a6c421ffd8908c4cd5bbedc4014c1b876488e8"},
-    {file = "Cython-0.29.21-cp34-cp34m-win_amd64.whl", hash = "sha256:c4b78356074fcaac04ecb4de289f11d506e438859877670992ece11f9c90f37b"},
-    {file = "Cython-0.29.21-cp35-cp35m-macosx_10_6_intel.whl", hash = "sha256:b2f9172e4d6358f33ecce6a4339b5960f9f83eab67ea244baa812737793826b7"},
-    {file = "Cython-0.29.21-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:856c7fb31d247ce713d60116375e1f8153d0291ab5e92cca7d8833a524ba9991"},
-    {file = "Cython-0.29.21-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:715294cd2246b39a8edca464a8366eb635f17213e4a6b9e74e52d8b877a8cb63"},
-    {file = "Cython-0.29.21-cp35-cp35m-win32.whl", hash = "sha256:23f3a00b843a19de8bb4468b087db5b413a903213f67188729782488d67040e0"},
-    {file = "Cython-0.29.21-cp35-cp35m-win_amd64.whl", hash = "sha256:ccb77faeaad99e99c6c444d04862c6cf604204fe0a07d4c8f9cbf2c9012d7d5a"},
-    {file = "Cython-0.29.21-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:e272ed97d20b026f4f25a012b25d7d7672a60e4f72b9ca385239d693cd91b2d5"},
-    {file = "Cython-0.29.21-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:8c6e25e9cc4961bb2abb1777c6fa9d0fa2d9b014beb3276cebe69996ff162b78"},
-    {file = "Cython-0.29.21-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:57ead89128dee9609119c93d3926c7a2add451453063147900408a50144598c6"},
-    {file = "Cython-0.29.21-cp36-cp36m-win32.whl", hash = "sha256:0e25c209c75df8785480dcef85db3d36c165dbc0f4c503168e8763eb735704f2"},
-    {file = "Cython-0.29.21-cp36-cp36m-win_amd64.whl", hash = "sha256:a0674f246ad5e1571ef29d4c5ec1d6ecabe9e6c424ad0d6fee46b914d5d24d69"},
-    {file = "Cython-0.29.21-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:5da187bebe38030325e1c0b5b8a804d489410be2d384c0ef3ba39493c67eb51e"},
-    {file = "Cython-0.29.21-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:9ce5e5209f8406ffc2b058b1293cce7a954911bb7991e623564d489197c9ba30"},
-    {file = "Cython-0.29.21-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:5e545a48f919e40079b0efe7b0e081c74b96f9ef25b9c1ff4cdbd95764426b58"},
-    {file = "Cython-0.29.21-cp37-cp37m-win32.whl", hash = "sha256:c8435959321cf8aec867bbad54b83b7fb8343204b530d85d9ea7a1f5329d5ac2"},
-    {file = "Cython-0.29.21-cp37-cp37m-win_amd64.whl", hash = "sha256:540b3bee0711aac2e99bda4fa0a46dbcd8c74941666bfc1ef9236b1a64eeffd9"},
-    {file = "Cython-0.29.21-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:93f5fed1c9445fb7afe20450cdaf94b0e0356d47cc75008105be89c6a2e417b1"},
-    {file = "Cython-0.29.21-cp38-cp38-manylinux1_i686.whl", hash = "sha256:9207fdedc7e789a3dcaca628176b80c82fbed9ae0997210738cbb12536a56699"},
-    {file = "Cython-0.29.21-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:603b9f1b8e93e8b494d3e89320c410679e21018e48b6cbc77280f5db71f17dc0"},
-    {file = "Cython-0.29.21-cp38-cp38-win32.whl", hash = "sha256:473df5d5e400444a36ed81c6596f56a5b52a3481312d0a48d68b777790f730ae"},
-    {file = "Cython-0.29.21-cp38-cp38-win_amd64.whl", hash = "sha256:b8a8a31b9e8860634adbca30fea1d0c7f08e208b3d7611f3e580e5f20992e5d7"},
-    {file = "Cython-0.29.21-py2.py3-none-any.whl", hash = "sha256:5c4276fdcbccdf1e3c1756c7aeb8395e9a36874fa4d30860e7694f43d325ae13"},
-    {file = "Cython-0.29.21.tar.gz", hash = "sha256:e57acb89bd55943c8d8bf813763d20b9099cc7165c0f16b707631a7654be9cad"},
+    {file = "Cython-0.29.23-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:ff885f18d169759b57f116d3956e45cd2b9cba989fde348bba091544c668dc11"},
+    {file = "Cython-0.29.23-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:3b29224eb62309a10819d923dc6262f769e4f3facfee3cd06372c355e5b38b33"},
+    {file = "Cython-0.29.23-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:355a6e768d91e21fbf477b61881bab64b7a2da386a166898997bccefd532cf5d"},
+    {file = "Cython-0.29.23-cp27-cp27m-win32.whl", hash = "sha256:2af52d312e96b38ded38b34d06e22685c226b1b0e58278bd27209f5d2385d115"},
+    {file = "Cython-0.29.23-cp27-cp27m-win_amd64.whl", hash = "sha256:519fccf526d26b377e1db22f22aa44889b28bc5833ec106588cb13557e8ba2da"},
+    {file = "Cython-0.29.23-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:625a16103770fd92b487b701fb0c07e5790b080f40fa11ce572a2d56d9e9fcca"},
+    {file = "Cython-0.29.23-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:7b7a766726d207d7cd57aff0fcb4b35ce042d3cc88a421fcdb45eeb61a5b9d12"},
+    {file = "Cython-0.29.23-cp34-cp34m-win32.whl", hash = "sha256:474c1a29ab43e29d990df279e2cf6aa96baa9208f5cd4bc76ac87ffcdf1e2945"},
+    {file = "Cython-0.29.23-cp34-cp34m-win_amd64.whl", hash = "sha256:f4aca6bffb1c1c3c4ada3347d0b162a699c18a66e097ee08b63b3a35118fdfcc"},
+    {file = "Cython-0.29.23-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:58dc06871bfdb0592542d779714fe9f918e11ba20ac07757dd63b198bdc704fe"},
+    {file = "Cython-0.29.23-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:2a3bbce689a2fddb85aa66712d93875c99bf7f64ac82b1d149ecce522a7a4e0c"},
+    {file = "Cython-0.29.23-cp35-cp35m-win32.whl", hash = "sha256:3ef530f975e3a760e7282fce2a25f900fa63f96d17321b4aa5f5542eb9859cdf"},
+    {file = "Cython-0.29.23-cp35-cp35m-win_amd64.whl", hash = "sha256:ef21c51350462160456eb71df31b0869e5141e940f22c61c358bdb6e3ebc3388"},
+    {file = "Cython-0.29.23-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:20402ef316393168909926ab21848aa6e08e39bed5003b657139774e66166cd0"},
+    {file = "Cython-0.29.23-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:4858043ac5f96a8f0277cf63760bb39b9521c1f897678cf1d22423f3e758f4ed"},
+    {file = "Cython-0.29.23-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:4d7c3b0882d8757c601eaf288fc0d321d5c7ac6c3afb8c42eddf9325a3419cf5"},
+    {file = "Cython-0.29.23-cp36-cp36m-win32.whl", hash = "sha256:37ff66039e3d138ec968ee1d1e12441fa5fb4e6a9c5458bc3c3a232f01be4a7d"},
+    {file = "Cython-0.29.23-cp36-cp36m-win_amd64.whl", hash = "sha256:5be3ae3189cf7d0e9bbeafb854496dc7030c6f6a5602d809435fab8223543a41"},
+    {file = "Cython-0.29.23-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b0699f0dc90181f2458fdb8170455e7798a309e18f41379eda7a2dc8c7aadee0"},
+    {file = "Cython-0.29.23-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:41cd0dd2ff5d78466e73409db509887a84449b400074d4f217980cedbb18e4be"},
+    {file = "Cython-0.29.23-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:aa3bb0928fb2aa3a8828801eb8b29af2261c199f805ae835467489e2bdd00372"},
+    {file = "Cython-0.29.23-cp37-cp37m-win32.whl", hash = "sha256:20cb50d9fede8029bdb50875458f07a27f909289aeed4cdb9c19544dd9a9bc45"},
+    {file = "Cython-0.29.23-cp37-cp37m-win_amd64.whl", hash = "sha256:c4b82461edbbcf90f19b319006345b77474a2d7514e1476d49a14bbd55d6b797"},
+    {file = "Cython-0.29.23-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:794e3df0b57e16bce7583ac909126f4cb381fe566adadb20484d89095855eedb"},
+    {file = "Cython-0.29.23-cp38-cp38-manylinux1_i686.whl", hash = "sha256:0c4b9f7e3aa004cf3f364e3e772f55fec5740485bafea99d1f13bdc9bbd8a545"},
+    {file = "Cython-0.29.23-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:ceccc03b633113ede1f14ad914a6db5c278ce108c8ddb308a5c01c1567d8a02a"},
+    {file = "Cython-0.29.23-cp38-cp38-win32.whl", hash = "sha256:4b0bcf2e06a9063fc78c3243ed4003228375d532ef13b9e5d7183be8f0a52cf5"},
+    {file = "Cython-0.29.23-cp38-cp38-win_amd64.whl", hash = "sha256:5a6792153b728a0240e55bbb5b643f4f7e45c76319e03abf15bf367471ea1d1a"},
+    {file = "Cython-0.29.23-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:a8eed9c82e8fe07b8a8ffbd36018871a17458903fc25c9d015f37b54513a3efd"},
+    {file = "Cython-0.29.23-cp39-cp39-manylinux1_i686.whl", hash = "sha256:266459c7e48fe3c6c492b297e4033e42d4c6863cc1a1ff7cc4034949fc574fa6"},
+    {file = "Cython-0.29.23-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:4b6824b58d4373224fc76ee8bee6b35c2d17c91a1ed0fa67b88440f63daebe50"},
+    {file = "Cython-0.29.23-cp39-cp39-win32.whl", hash = "sha256:7d6a33c8a11f05f698e215bfdb837f32c27f63c20f3af863557ed91c748dc2be"},
+    {file = "Cython-0.29.23-cp39-cp39-win_amd64.whl", hash = "sha256:2365f3b5e6451b6bc6dcd262230656f4ade1d862ec2f6c22154deebef37c08b6"},
+    {file = "Cython-0.29.23-py2.py3-none-any.whl", hash = "sha256:282263628c5d601b313d5920f7b6d7e08c7fedbddacd080c4858aa04d86b6b4b"},
+    {file = "Cython-0.29.23.tar.gz", hash = "sha256:6a0d31452f0245daacb14c979c77e093eb1a546c760816b5eed0047686baad8e"},
 ]
 fasteners = [
-    {file = "fasteners-0.15-py2.py3-none-any.whl", hash = "sha256:007e4d2b2d4a10093f67e932e5166722d2eab83b77724156e92ad013c6226574"},
-    {file = "fasteners-0.15.tar.gz", hash = "sha256:3a176da6b70df9bb88498e1a18a9e4a8579ed5b9141207762368a1017bf8f5ef"},
-]
-future = [
-    {file = "future-0.18.2.tar.gz", hash = "sha256:b1bead90b70cf6ec3f0710ae53a525360fa360d306a86583adc6bf83a4db537d"},
+    {file = "fasteners-0.16-py2.py3-none-any.whl", hash = "sha256:74b6847e0a6bb3b56c8511af8e24c40e4cf7a774dfff5b251c260ed338096a4b"},
+    {file = "fasteners-0.16.tar.gz", hash = "sha256:c995d8c26b017c5d6a6de9ad29a0f9cdd57de61ae1113d28fac26622b06a0933"},
 ]
 glfw = [
-    {file = "glfw-2.0.0-py2.py27.py3.py30.py31.py32.py33.py34.py35.py36.py37.py38-none-macosx_10_6_intel.whl", hash = "sha256:76cddb57ebfe36812cca3d75267f79e94eee449f7a001429cd67923af0f4740d"},
-    {file = "glfw-2.0.0-py2.py27.py3.py30.py31.py32.py33.py34.py35.py36.py37.py38-none-manylinux2010_i686.whl", hash = "sha256:c60c375257b95af14aa3561b420879eb5d313a55195116337be4a345604ec668"},
-    {file = "glfw-2.0.0-py2.py27.py3.py30.py31.py32.py33.py34.py35.py36.py37.py38-none-manylinux2010_x86_64.whl", hash = "sha256:ef4f4c1a24d9955244563db0b6da46d8fafb457a32f295a029087aafbfcdc898"},
-    {file = "glfw-2.0.0-py2.py27.py3.py30.py31.py32.py33.py34.py35.py36.py37.py38-none-manylinux2014_x86_64.whl", hash = "sha256:92d91a60f5a93ce7a4cd124ad73fd2a7b13dc17fced7608f3f9d8202ca61e495"},
-    {file = "glfw-2.0.0-py2.py27.py3.py30.py31.py32.py33.py34.py35.py36.py37.py38-none-win32.whl", hash = "sha256:ee7ceacbb8688f0f5c05fa7f88c1071698ef896a192a8acc0f7561e8949f8e85"},
-    {file = "glfw-2.0.0-py2.py27.py3.py30.py31.py32.py33.py34.py35.py36.py37.py38-none-win_amd64.whl", hash = "sha256:7f40ee7e34f41e63ef775b46b5d199a312c971f97a0c92d5de341e1b45b1bf00"},
-    {file = "glfw-2.0.0.tar.gz", hash = "sha256:7a330b0abd4f5367510883f11e4f2b31c2d0dcc96cac9669107e1bd3426e98ce"},
+    {file = "glfw-2.1.0-py2.py27.py3.py30.py31.py32.py33.py34.py35.py36.py37.py38-none-macosx_10_6_intel.whl", hash = "sha256:5499f1daa773f6464eb16ac4c8f64c3533408d73863987c4b9131d0a20648f1f"},
+    {file = "glfw-2.1.0-py2.py27.py3.py30.py31.py32.py33.py34.py35.py36.py37.py38-none-manylinux2010_i686.whl", hash = "sha256:b0c440534224a371becafb96019f5f4340f8f65d5c2bcbfad537c3f40ca8e307"},
+    {file = "glfw-2.1.0-py2.py27.py3.py30.py31.py32.py33.py34.py35.py36.py37.py38-none-manylinux2010_x86_64.whl", hash = "sha256:12953e121ede2e5980036cd1cc667fa7918b67799b903699616439e3deab6803"},
+    {file = "glfw-2.1.0-py2.py27.py3.py30.py31.py32.py33.py34.py35.py36.py37.py38-none-manylinux2014_x86_64.whl", hash = "sha256:92e95fcf2088c371ce7411e8f34598cccbcb46c88fc7d91a5f19d15b94e961c1"},
+    {file = "glfw-2.1.0-py2.py27.py3.py30.py31.py32.py33.py34.py35.py36.py37.py38-none-win32.whl", hash = "sha256:83db9be8291a3e7ae121a443f874a3e3f3ae75c3e52e66f231db1818553811c4"},
+    {file = "glfw-2.1.0-py2.py27.py3.py30.py31.py32.py33.py34.py35.py36.py37.py38-none-win_amd64.whl", hash = "sha256:8f40d12e9c7599c4d10b74e74ef79c8c7175d966a7fc35e6210b951004e48cf7"},
+    {file = "glfw-2.1.0.tar.gz", hash = "sha256:464156ef108801b03f152553622f46b4b2acb080bdab91398dd22a84dfd379b3"},
 ]
 gym = [
-    {file = "gym-0.17.3.tar.gz", hash = "sha256:96a7dd4e9cdb39e30c7a79e5773570fd9408f7fdb58c714c293cfbb314818eb6"},
+    {file = "gym-0.18.3.tar.gz", hash = "sha256:81a3e3fbf7fcf57c8cf98f7e22d1bdd5815f3824d9c148a7eb42420d3d642967"},
 ]
 imageio = [
     {file = "imageio-2.9.0-py3-none-any.whl", hash = "sha256:3604d751f03002e8e0e7650aa71d8d9148144a87daf17cb1f3228e80747f2e6b"},
     {file = "imageio-2.9.0.tar.gz", hash = "sha256:52ddbaeca2dccf53ba2d6dec5676ca7bc3b2403ef8b37f7da78b7654bb3e10f0"},
 ]
 importlib-metadata = [
-    {file = "importlib_metadata-2.0.0-py2.py3-none-any.whl", hash = "sha256:cefa1a2f919b866c5beb7c9f7b0ebb4061f30a8a9bf16d609b000e2dfaceb9c3"},
-    {file = "importlib_metadata-2.0.0.tar.gz", hash = "sha256:77a540690e24b0305878c37ffd421785a6f7e53c8b5720d211b211de8d0e95da"},
-]
-monotonic = [
-    {file = "monotonic-1.5-py2.py3-none-any.whl", hash = "sha256:552a91f381532e33cbd07c6a2655a21908088962bb8fa7239ecbcc6ad1140cc7"},
-    {file = "monotonic-1.5.tar.gz", hash = "sha256:23953d55076df038541e648a53676fb24980f7a1be290cdda21300b3bc21dfb0"},
+    {file = "importlib_metadata-4.4.0-py3-none-any.whl", hash = "sha256:960d52ba7c21377c990412aca380bf3642d734c2eaab78a2c39319f67c6a5786"},
+    {file = "importlib_metadata-4.4.0.tar.gz", hash = "sha256:e592faad8de1bda9fe920cf41e15261e7131bcf266c30306eec00e8e225c1dd5"},
 ]
 more-itertools = [
-    {file = "more-itertools-8.5.0.tar.gz", hash = "sha256:6f83822ae94818eae2612063a5101a7311e68ae8002005b5e05f03fd74a86a20"},
-    {file = "more_itertools-8.5.0-py3-none-any.whl", hash = "sha256:9b30f12df9393f0d28af9210ff8efe48d10c94f73e5daf886f10c4b0b0b4f03c"},
+    {file = "more-itertools-8.8.0.tar.gz", hash = "sha256:83f0308e05477c68f56ea3a888172c78ed5d5b3c282addb67508e7ba6c8f813a"},
+    {file = "more_itertools-8.8.0-py3-none-any.whl", hash = "sha256:2cf89ec599962f2ddc4d568a05defc40e0a587fbc10d5989713638864c36be4d"},
 ]
 mujoco-py = [
     {file = "mujoco-py-2.0.2.13.tar.gz", hash = "sha256:d6ae66276b565af9063597fda70683a89c7356290f5ac3961b794ee90ec50eea"},
 ]
 numpy = [
-    {file = "numpy-1.19.2-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:b594f76771bc7fc8a044c5ba303427ee67c17a09b36e1fa32bde82f5c419d17a"},
-    {file = "numpy-1.19.2-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:e6ddbdc5113628f15de7e4911c02aed74a4ccff531842c583e5032f6e5a179bd"},
-    {file = "numpy-1.19.2-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:3733640466733441295b0d6d3dcbf8e1ffa7e897d4d82903169529fd3386919a"},
-    {file = "numpy-1.19.2-cp36-cp36m-manylinux2010_i686.whl", hash = "sha256:4339741994c775396e1a274dba3609c69ab0f16056c1077f18979bec2a2c2e6e"},
-    {file = "numpy-1.19.2-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:7c6646314291d8f5ea900a7ea9c4261f834b5b62159ba2abe3836f4fa6705526"},
-    {file = "numpy-1.19.2-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:7118f0a9f2f617f921ec7d278d981244ba83c85eea197be7c5a4f84af80a9c3c"},
-    {file = "numpy-1.19.2-cp36-cp36m-win32.whl", hash = "sha256:9a3001248b9231ed73894c773142658bab914645261275f675d86c290c37f66d"},
-    {file = "numpy-1.19.2-cp36-cp36m-win_amd64.whl", hash = "sha256:967c92435f0b3ba37a4257c48b8715b76741410467e2bdb1097e8391fccfae15"},
-    {file = "numpy-1.19.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:d526fa58ae4aead839161535d59ea9565863bb0b0bdb3cc63214613fb16aced4"},
-    {file = "numpy-1.19.2-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:eb25c381d168daf351147713f49c626030dcff7a393d5caa62515d415a6071d8"},
-    {file = "numpy-1.19.2-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:62139af94728d22350a571b7c82795b9d59be77fc162414ada6c8b6a10ef5d02"},
-    {file = "numpy-1.19.2-cp37-cp37m-manylinux2010_i686.whl", hash = "sha256:0c66da1d202c52051625e55a249da35b31f65a81cb56e4c69af0dfb8fb0125bf"},
-    {file = "numpy-1.19.2-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:2117536e968abb7357d34d754e3733b0d7113d4c9f1d921f21a3d96dec5ff716"},
-    {file = "numpy-1.19.2-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:54045b198aebf41bf6bf4088012777c1d11703bf74461d70cd350c0af2182e45"},
-    {file = "numpy-1.19.2-cp37-cp37m-win32.whl", hash = "sha256:aba1d5daf1144b956bc87ffb87966791f5e9f3e1f6fab3d7f581db1f5b598f7a"},
-    {file = "numpy-1.19.2-cp37-cp37m-win_amd64.whl", hash = "sha256:addaa551b298052c16885fc70408d3848d4e2e7352de4e7a1e13e691abc734c1"},
-    {file = "numpy-1.19.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:58d66a6b3b55178a1f8a5fe98df26ace76260a70de694d99577ddeab7eaa9a9d"},
-    {file = "numpy-1.19.2-cp38-cp38-manylinux1_i686.whl", hash = "sha256:59f3d687faea7a4f7f93bd9665e5b102f32f3fa28514f15b126f099b7997203d"},
-    {file = "numpy-1.19.2-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:cebd4f4e64cfe87f2039e4725781f6326a61f095bc77b3716502bed812b385a9"},
-    {file = "numpy-1.19.2-cp38-cp38-manylinux2010_i686.whl", hash = "sha256:c35a01777f81e7333bcf276b605f39c872e28295441c265cd0c860f4b40148c1"},
-    {file = "numpy-1.19.2-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:d7ac33585e1f09e7345aa902c281bd777fdb792432d27fca857f39b70e5dd31c"},
-    {file = "numpy-1.19.2-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:04c7d4ebc5ff93d9822075ddb1751ff392a4375e5885299445fcebf877f179d5"},
-    {file = "numpy-1.19.2-cp38-cp38-win32.whl", hash = "sha256:51ee93e1fac3fe08ef54ff1c7f329db64d8a9c5557e6c8e908be9497ac76374b"},
-    {file = "numpy-1.19.2-cp38-cp38-win_amd64.whl", hash = "sha256:1669ec8e42f169ff715a904c9b2105b6640f3f2a4c4c2cb4920ae8b2785dac65"},
-    {file = "numpy-1.19.2-pp36-pypy36_pp73-manylinux2010_x86_64.whl", hash = "sha256:0bfd85053d1e9f60234f28f63d4a5147ada7f432943c113a11afcf3e65d9d4c8"},
-    {file = "numpy-1.19.2.zip", hash = "sha256:0d310730e1e793527065ad7dde736197b705d0e4c9999775f212b03c44a8484c"},
+    {file = "numpy-1.20.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:70eb5808127284c4e5c9e836208e09d685a7978b6a216db85960b1a112eeace8"},
+    {file = "numpy-1.20.3-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:6ca2b85a5997dabc38301a22ee43c82adcb53ff660b89ee88dded6b33687e1d8"},
+    {file = "numpy-1.20.3-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:c5bf0e132acf7557fc9bb8ded8b53bbbbea8892f3c9a1738205878ca9434206a"},
+    {file = "numpy-1.20.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:db250fd3e90117e0312b611574cd1b3f78bec046783195075cbd7ba9c3d73f16"},
+    {file = "numpy-1.20.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:637d827248f447e63585ca3f4a7d2dfaa882e094df6cfa177cc9cf9cd6cdf6d2"},
+    {file = "numpy-1.20.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:8b7bb4b9280da3b2856cb1fc425932f46fba609819ee1c62256f61799e6a51d2"},
+    {file = "numpy-1.20.3-cp37-cp37m-win32.whl", hash = "sha256:67d44acb72c31a97a3d5d33d103ab06d8ac20770e1c5ad81bdb3f0c086a56cf6"},
+    {file = "numpy-1.20.3-cp37-cp37m-win_amd64.whl", hash = "sha256:43909c8bb289c382170e0282158a38cf306a8ad2ff6dfadc447e90f9961bef43"},
+    {file = "numpy-1.20.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:f1452578d0516283c87608a5a5548b0cdde15b99650efdfd85182102ef7a7c17"},
+    {file = "numpy-1.20.3-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:6e51534e78d14b4a009a062641f465cfaba4fdcb046c3ac0b1f61dd97c861b1b"},
+    {file = "numpy-1.20.3-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:e515c9a93aebe27166ec9593411c58494fa98e5fcc219e47260d9ab8a1cc7f9f"},
+    {file = "numpy-1.20.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c1c09247ccea742525bdb5f4b5ceeacb34f95731647fe55774aa36557dbb5fa4"},
+    {file = "numpy-1.20.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:66fbc6fed94a13b9801fb70b96ff30605ab0a123e775a5e7a26938b717c5d71a"},
+    {file = "numpy-1.20.3-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:ea9cff01e75a956dbee133fa8e5b68f2f92175233de2f88de3a682dd94deda65"},
+    {file = "numpy-1.20.3-cp38-cp38-win32.whl", hash = "sha256:f39a995e47cb8649673cfa0579fbdd1cdd33ea497d1728a6cb194d6252268e48"},
+    {file = "numpy-1.20.3-cp38-cp38-win_amd64.whl", hash = "sha256:1676b0a292dd3c99e49305a16d7a9f42a4ab60ec522eac0d3dd20cdf362ac010"},
+    {file = "numpy-1.20.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:830b044f4e64a76ba71448fce6e604c0fc47a0e54d8f6467be23749ac2cbd2fb"},
+    {file = "numpy-1.20.3-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:55b745fca0a5ab738647d0e4db099bd0a23279c32b31a783ad2ccea729e632df"},
+    {file = "numpy-1.20.3-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:5d050e1e4bc9ddb8656d7b4f414557720ddcca23a5b88dd7cff65e847864c400"},
+    {file = "numpy-1.20.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a9c65473ebc342715cb2d7926ff1e202c26376c0dcaaee85a1fd4b8d8c1d3b2f"},
+    {file = "numpy-1.20.3-cp39-cp39-win32.whl", hash = "sha256:16f221035e8bd19b9dc9a57159e38d2dd060b48e93e1d843c49cb370b0f415fd"},
+    {file = "numpy-1.20.3-cp39-cp39-win_amd64.whl", hash = "sha256:6690080810f77485667bfbff4f69d717c3be25e5b11bb2073e76bb3f578d99b4"},
+    {file = "numpy-1.20.3-pp37-pypy37_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:4e465afc3b96dbc80cf4a5273e5e2b1e3451286361b4af70ce1adb2984d392f9"},
+    {file = "numpy-1.20.3.zip", hash = "sha256:e55185e51b18d788e49fe8305fd73ef4470596b33fc2c1ceb304566b99c71a69"},
 ]
 pillow = [
-    {file = "Pillow-7.2.0-cp35-cp35m-macosx_10_10_intel.whl", hash = "sha256:1ca594126d3c4def54babee699c055a913efb01e106c309fa6b04405d474d5ae"},
-    {file = "Pillow-7.2.0-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:c92302a33138409e8f1ad16731568c55c9053eee71bb05b6b744067e1b62380f"},
-    {file = "Pillow-7.2.0-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:8dad18b69f710bf3a001d2bf3afab7c432785d94fcf819c16b5207b1cfd17d38"},
-    {file = "Pillow-7.2.0-cp35-cp35m-manylinux2014_aarch64.whl", hash = "sha256:431b15cffbf949e89df2f7b48528be18b78bfa5177cb3036284a5508159492b5"},
-    {file = "Pillow-7.2.0-cp35-cp35m-win32.whl", hash = "sha256:09d7f9e64289cb40c2c8d7ad674b2ed6105f55dc3b09aa8e4918e20a0311e7ad"},
-    {file = "Pillow-7.2.0-cp35-cp35m-win_amd64.whl", hash = "sha256:0295442429645fa16d05bd567ef5cff178482439c9aad0411d3f0ce9b88b3a6f"},
-    {file = "Pillow-7.2.0-cp36-cp36m-macosx_10_10_x86_64.whl", hash = "sha256:ec29604081f10f16a7aea809ad42e27764188fc258b02259a03a8ff7ded3808d"},
-    {file = "Pillow-7.2.0-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:612cfda94e9c8346f239bf1a4b082fdd5c8143cf82d685ba2dba76e7adeeb233"},
-    {file = "Pillow-7.2.0-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:0a80dd307a5d8440b0a08bd7b81617e04d870e40a3e46a32d9c246e54705e86f"},
-    {file = "Pillow-7.2.0-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:06aba4169e78c439d528fdeb34762c3b61a70813527a2c57f0540541e9f433a8"},
-    {file = "Pillow-7.2.0-cp36-cp36m-win32.whl", hash = "sha256:f7e30c27477dffc3e85c2463b3e649f751789e0f6c8456099eea7ddd53be4a8a"},
-    {file = "Pillow-7.2.0-cp36-cp36m-win_amd64.whl", hash = "sha256:ffe538682dc19cc542ae7c3e504fdf54ca7f86fb8a135e59dd6bc8627eae6cce"},
-    {file = "Pillow-7.2.0-cp37-cp37m-macosx_10_10_x86_64.whl", hash = "sha256:94cf49723928eb6070a892cb39d6c156f7b5a2db4e8971cb958f7b6b104fb4c4"},
-    {file = "Pillow-7.2.0-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:6edb5446f44d901e8683ffb25ebdfc26988ee813da3bf91e12252b57ac163727"},
-    {file = "Pillow-7.2.0-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:52125833b070791fcb5710fabc640fc1df07d087fc0c0f02d3661f76c23c5b8b"},
-    {file = "Pillow-7.2.0-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:9ad7f865eebde135d526bb3163d0b23ffff365cf87e767c649550964ad72785d"},
-    {file = "Pillow-7.2.0-cp37-cp37m-win32.whl", hash = "sha256:c79f9c5fb846285f943aafeafda3358992d64f0ef58566e23484132ecd8d7d63"},
-    {file = "Pillow-7.2.0-cp37-cp37m-win_amd64.whl", hash = "sha256:d350f0f2c2421e65fbc62690f26b59b0bcda1b614beb318c81e38647e0f673a1"},
-    {file = "Pillow-7.2.0-cp38-cp38-macosx_10_10_x86_64.whl", hash = "sha256:6d7741e65835716ceea0fd13a7d0192961212fd59e741a46bbed7a473c634ed6"},
-    {file = "Pillow-7.2.0-cp38-cp38-manylinux1_i686.whl", hash = "sha256:edf31f1150778abd4322444c393ab9c7bd2af271dd4dafb4208fb613b1f3cdc9"},
-    {file = "Pillow-7.2.0-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:d08b23fdb388c0715990cbc06866db554e1822c4bdcf6d4166cf30ac82df8c41"},
-    {file = "Pillow-7.2.0-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:5e51ee2b8114def244384eda1c82b10e307ad9778dac5c83fb0943775a653cd8"},
-    {file = "Pillow-7.2.0-cp38-cp38-win32.whl", hash = "sha256:725aa6cfc66ce2857d585f06e9519a1cc0ef6d13f186ff3447ab6dff0a09bc7f"},
-    {file = "Pillow-7.2.0-cp38-cp38-win_amd64.whl", hash = "sha256:a060cf8aa332052df2158e5a119303965be92c3da6f2d93b6878f0ebca80b2f6"},
-    {file = "Pillow-7.2.0-pp36-pypy36_pp73-macosx_10_10_x86_64.whl", hash = "sha256:9c87ef410a58dd54b92424ffd7e28fd2ec65d2f7fc02b76f5e9b2067e355ebf6"},
-    {file = "Pillow-7.2.0-pp36-pypy36_pp73-manylinux2010_x86_64.whl", hash = "sha256:e901964262a56d9ea3c2693df68bc9860b8bdda2b04768821e4c44ae797de117"},
-    {file = "Pillow-7.2.0-pp36-pypy36_pp73-win32.whl", hash = "sha256:25930fadde8019f374400f7986e8404c8b781ce519da27792cbe46eabec00c4d"},
-    {file = "Pillow-7.2.0.tar.gz", hash = "sha256:97f9e7953a77d5a70f49b9a48da7776dc51e9b738151b22dacf101641594a626"},
+    {file = "Pillow-8.2.0-cp36-cp36m-macosx_10_10_x86_64.whl", hash = "sha256:dc38f57d8f20f06dd7c3161c59ca2c86893632623f33a42d592f097b00f720a9"},
+    {file = "Pillow-8.2.0-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:a013cbe25d20c2e0c4e85a9daf438f85121a4d0344ddc76e33fd7e3965d9af4b"},
+    {file = "Pillow-8.2.0-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:8bb1e155a74e1bfbacd84555ea62fa21c58e0b4e7e6b20e4447b8d07990ac78b"},
+    {file = "Pillow-8.2.0-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:c5236606e8570542ed424849f7852a0ff0bce2c4c8d0ba05cc202a5a9c97dee9"},
+    {file = "Pillow-8.2.0-cp36-cp36m-win32.whl", hash = "sha256:12e5e7471f9b637762453da74e390e56cc43e486a88289995c1f4c1dc0bfe727"},
+    {file = "Pillow-8.2.0-cp36-cp36m-win_amd64.whl", hash = "sha256:5afe6b237a0b81bd54b53f835a153770802f164c5570bab5e005aad693dab87f"},
+    {file = "Pillow-8.2.0-cp37-cp37m-macosx_10_10_x86_64.whl", hash = "sha256:cb7a09e173903541fa888ba010c345893cd9fc1b5891aaf060f6ca77b6a3722d"},
+    {file = "Pillow-8.2.0-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:0d19d70ee7c2ba97631bae1e7d4725cdb2ecf238178096e8c82ee481e189168a"},
+    {file = "Pillow-8.2.0-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:083781abd261bdabf090ad07bb69f8f5599943ddb539d64497ed021b2a67e5a9"},
+    {file = "Pillow-8.2.0-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:c6b39294464b03457f9064e98c124e09008b35a62e3189d3513e5148611c9388"},
+    {file = "Pillow-8.2.0-cp37-cp37m-win32.whl", hash = "sha256:01425106e4e8cee195a411f729cff2a7d61813b0b11737c12bd5991f5f14bcd5"},
+    {file = "Pillow-8.2.0-cp37-cp37m-win_amd64.whl", hash = "sha256:3b570f84a6161cf8865c4e08adf629441f56e32f180f7aa4ccbd2e0a5a02cba2"},
+    {file = "Pillow-8.2.0-cp38-cp38-macosx_10_10_x86_64.whl", hash = "sha256:031a6c88c77d08aab84fecc05c3cde8414cd6f8406f4d2b16fed1e97634cc8a4"},
+    {file = "Pillow-8.2.0-cp38-cp38-manylinux1_i686.whl", hash = "sha256:66cc56579fd91f517290ab02c51e3a80f581aba45fd924fcdee01fa06e635812"},
+    {file = "Pillow-8.2.0-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:6c32cc3145928c4305d142ebec682419a6c0a8ce9e33db900027ddca1ec39178"},
+    {file = "Pillow-8.2.0-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:624b977355cde8b065f6d51b98497d6cd5fbdd4f36405f7a8790e3376125e2bb"},
+    {file = "Pillow-8.2.0-cp38-cp38-win32.whl", hash = "sha256:5cbf3e3b1014dddc45496e8cf38b9f099c95a326275885199f427825c6522232"},
+    {file = "Pillow-8.2.0-cp38-cp38-win_amd64.whl", hash = "sha256:463822e2f0d81459e113372a168f2ff59723e78528f91f0bd25680ac185cf797"},
+    {file = "Pillow-8.2.0-cp39-cp39-macosx_10_10_x86_64.whl", hash = "sha256:95d5ef984eff897850f3a83883363da64aae1000e79cb3c321915468e8c6add5"},
+    {file = "Pillow-8.2.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:b91c36492a4bbb1ee855b7d16fe51379e5f96b85692dc8210831fbb24c43e484"},
+    {file = "Pillow-8.2.0-cp39-cp39-manylinux1_i686.whl", hash = "sha256:d68cb92c408261f806b15923834203f024110a2e2872ecb0bd2a110f89d3c602"},
+    {file = "Pillow-8.2.0-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:f217c3954ce5fd88303fc0c317af55d5e0204106d86dea17eb8205700d47dec2"},
+    {file = "Pillow-8.2.0-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:5b70110acb39f3aff6b74cf09bb4169b167e2660dabc304c1e25b6555fa781ef"},
+    {file = "Pillow-8.2.0-cp39-cp39-win32.whl", hash = "sha256:a7d5e9fad90eff8f6f6106d3b98b553a88b6f976e51fce287192a5d2d5363713"},
+    {file = "Pillow-8.2.0-cp39-cp39-win_amd64.whl", hash = "sha256:238c197fc275b475e87c1453b05b467d2d02c2915fdfdd4af126145ff2e4610c"},
+    {file = "Pillow-8.2.0-pp36-pypy36_pp73-macosx_10_10_x86_64.whl", hash = "sha256:0e04d61f0064b545b989126197930807c86bcbd4534d39168f4aa5fda39bb8f9"},
+    {file = "Pillow-8.2.0-pp36-pypy36_pp73-manylinux2010_i686.whl", hash = "sha256:63728564c1410d99e6d1ae8e3b810fe012bc440952168af0a2877e8ff5ab96b9"},
+    {file = "Pillow-8.2.0-pp36-pypy36_pp73-manylinux2010_x86_64.whl", hash = "sha256:c03c07ed32c5324939b19e36ae5f75c660c81461e312a41aea30acdd46f93a7c"},
+    {file = "Pillow-8.2.0-pp37-pypy37_pp73-macosx_10_10_x86_64.whl", hash = "sha256:4d98abdd6b1e3bf1a1cbb14c3895226816e666749ac040c4e2554231068c639b"},
+    {file = "Pillow-8.2.0-pp37-pypy37_pp73-manylinux2010_i686.whl", hash = "sha256:aac00e4bc94d1b7813fe882c28990c1bc2f9d0e1aa765a5f2b516e8a6a16a9e4"},
+    {file = "Pillow-8.2.0-pp37-pypy37_pp73-manylinux2010_x86_64.whl", hash = "sha256:22fd0f42ad15dfdde6c581347eaa4adb9a6fc4b865f90b23378aa7914895e120"},
+    {file = "Pillow-8.2.0-pp37-pypy37_pp73-win32.whl", hash = "sha256:e98eca29a05913e82177b3ba3d198b1728e164869c613d76d0de4bde6768a50e"},
+    {file = "Pillow-8.2.0-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:8b56553c0345ad6dcb2e9b433ae47d67f95fc23fe28a0bde15a120f25257e291"},
+    {file = "Pillow-8.2.0.tar.gz", hash = "sha256:a787ab10d7bb5494e5f76536ac460741788f1fbce851068d73a87ca7c35fc3e1"},
 ]
 pluggy = [
     {file = "pluggy-0.13.1-py2.py3-none-any.whl", hash = "sha256:966c145cd83c96502c3c3868f50408687b38434af77734af1e9ca461a4081d2d"},
     {file = "pluggy-0.13.1.tar.gz", hash = "sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0"},
 ]
 py = [
-    {file = "py-1.9.0-py2.py3-none-any.whl", hash = "sha256:366389d1db726cd2fcfc79732e75410e5fe4d31db13692115529d34069a043c2"},
-    {file = "py-1.9.0.tar.gz", hash = "sha256:9ca6883ce56b4e8da7e79ac18787889fa5206c79dcc67fb065376cd2fe03f342"},
+    {file = "py-1.10.0-py2.py3-none-any.whl", hash = "sha256:3b80836aa6d1feeaa108e046da6423ab8f6ceda6468545ae8d02d9d58d18818a"},
+    {file = "py-1.10.0.tar.gz", hash = "sha256:21b81bda15b66ef5e1a777a21c4dcd9c20ad3efd0b3f817e7a809035269e1bd3"},
 ]
 pycparser = [
     {file = "pycparser-2.20-py2.py3-none-any.whl", hash = "sha256:7582ad22678f0fcd81102833f60ef8d0e57288b6b5fb00323d101be910e35705"},
     {file = "pycparser-2.20.tar.gz", hash = "sha256:2d475327684562c3a96cc71adf7dc8c4f0565175cf86b6d7a404ff4c771f15f0"},
 ]
 pyglet = [
-    {file = "pyglet-1.5.0-py2.py3-none-any.whl", hash = "sha256:a42f599ebd0dc8113563041c402ae09be05cdcbc643bb1183785141ba3c3304e"},
-    {file = "pyglet-1.5.0.zip", hash = "sha256:6ea918985feddfa9bf0fcc01ffe9ff5849e7b6e832d9b2e03b9d2a36369cb6ee"},
+    {file = "pyglet-1.5.15-py3-none-any.whl", hash = "sha256:4401cc176580e4e17e2df8bbf7536f27e691327dc3f38f209a12f1859c70aed2"},
+    {file = "pyglet-1.5.15.zip", hash = "sha256:da9d8337388cedabf1f1c5dc21a45bb2b0e5327fba47f996c8573818c3dfa478"},
 ]
 pytest = [
     {file = "pytest-3.10.1-py2.py3-none-any.whl", hash = "sha256:3f193df1cfe1d1609d4c583838bea3d532b18d6160fd3f55c9447fdca30848ec"},
     {file = "pytest-3.10.1.tar.gz", hash = "sha256:e246cf173c01169b9617fc07264b7b1316e78d7a650055235d6d897bc80d9660"},
 ]
 scipy = [
-    {file = "scipy-1.5.2-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:cca9fce15109a36a0a9f9cfc64f870f1c140cb235ddf27fe0328e6afb44dfed0"},
-    {file = "scipy-1.5.2-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:1c7564a4810c1cd77fcdee7fa726d7d39d4e2695ad252d7c86c3ea9d85b7fb8f"},
-    {file = "scipy-1.5.2-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:07e52b316b40a4f001667d1ad4eb5f2318738de34597bd91537851365b6c61f1"},
-    {file = "scipy-1.5.2-cp36-cp36m-win32.whl", hash = "sha256:d56b10d8ed72ec1be76bf10508446df60954f08a41c2d40778bc29a3a9ad9bce"},
-    {file = "scipy-1.5.2-cp36-cp36m-win_amd64.whl", hash = "sha256:8e28e74b97fc8d6aa0454989db3b5d36fc27e69cef39a7ee5eaf8174ca1123cb"},
-    {file = "scipy-1.5.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:6e86c873fe1335d88b7a4bfa09d021f27a9e753758fd75f3f92d714aa4093768"},
-    {file = "scipy-1.5.2-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:a0afbb967fd2c98efad5f4c24439a640d39463282040a88e8e928db647d8ac3d"},
-    {file = "scipy-1.5.2-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:eecf40fa87eeda53e8e11d265ff2254729d04000cd40bae648e76ff268885d66"},
-    {file = "scipy-1.5.2-cp37-cp37m-win32.whl", hash = "sha256:315aa2165aca31375f4e26c230188db192ed901761390be908c9b21d8b07df62"},
-    {file = "scipy-1.5.2-cp37-cp37m-win_amd64.whl", hash = "sha256:ec5fe57e46828d034775b00cd625c4a7b5c7d2e354c3b258d820c6c72212a6ec"},
-    {file = "scipy-1.5.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:fc98f3eac993b9bfdd392e675dfe19850cc8c7246a8fd2b42443e506344be7d9"},
-    {file = "scipy-1.5.2-cp38-cp38-manylinux1_i686.whl", hash = "sha256:a785409c0fa51764766840185a34f96a0a93527a0ff0230484d33a8ed085c8f8"},
-    {file = "scipy-1.5.2-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:0a0e9a4e58a4734c2eba917f834b25b7e3b6dc333901ce7784fd31aefbd37b2f"},
-    {file = "scipy-1.5.2-cp38-cp38-win32.whl", hash = "sha256:dac09281a0eacd59974e24525a3bc90fa39b4e95177e638a31b14db60d3fa806"},
-    {file = "scipy-1.5.2-cp38-cp38-win_amd64.whl", hash = "sha256:92eb04041d371fea828858e4fff182453c25ae3eaa8782d9b6c32b25857d23bc"},
-    {file = "scipy-1.5.2.tar.gz", hash = "sha256:066c513d90eb3fd7567a9e150828d39111ebd88d3e924cdfc9f8ce19ab6f90c9"},
+    {file = "scipy-1.6.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a15a1f3fc0abff33e792d6049161b7795909b40b97c6cc2934ed54384017ab76"},
+    {file = "scipy-1.6.1-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:e79570979ccdc3d165456dd62041d9556fb9733b86b4b6d818af7a0afc15f092"},
+    {file = "scipy-1.6.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:a423533c55fec61456dedee7b6ee7dce0bb6bfa395424ea374d25afa262be261"},
+    {file = "scipy-1.6.1-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:33d6b7df40d197bdd3049d64e8e680227151673465e5d85723b3b8f6b15a6ced"},
+    {file = "scipy-1.6.1-cp37-cp37m-win32.whl", hash = "sha256:6725e3fbb47da428794f243864f2297462e9ee448297c93ed1dcbc44335feb78"},
+    {file = "scipy-1.6.1-cp37-cp37m-win_amd64.whl", hash = "sha256:5fa9c6530b1661f1370bcd332a1e62ca7881785cc0f80c0d559b636567fab63c"},
+    {file = "scipy-1.6.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:bd50daf727f7c195e26f27467c85ce653d41df4358a25b32434a50d8870fc519"},
+    {file = "scipy-1.6.1-cp38-cp38-manylinux1_i686.whl", hash = "sha256:f46dd15335e8a320b0fb4685f58b7471702234cba8bb3442b69a3e1dc329c345"},
+    {file = "scipy-1.6.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:0e5b0ccf63155d90da576edd2768b66fb276446c371b73841e3503be1d63fb5d"},
+    {file = "scipy-1.6.1-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:2481efbb3740977e3c831edfd0bd9867be26387cacf24eb5e366a6a374d3d00d"},
+    {file = "scipy-1.6.1-cp38-cp38-win32.whl", hash = "sha256:68cb4c424112cd4be886b4d979c5497fba190714085f46b8ae67a5e4416c32b4"},
+    {file = "scipy-1.6.1-cp38-cp38-win_amd64.whl", hash = "sha256:5f331eeed0297232d2e6eea51b54e8278ed8bb10b099f69c44e2558c090d06bf"},
+    {file = "scipy-1.6.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:0c8a51d33556bf70367452d4d601d1742c0e806cd0194785914daf19775f0e67"},
+    {file = "scipy-1.6.1-cp39-cp39-manylinux1_i686.whl", hash = "sha256:83bf7c16245c15bc58ee76c5418e46ea1811edcc2e2b03041b804e46084ab627"},
+    {file = "scipy-1.6.1-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:794e768cc5f779736593046c9714e0f3a5940bc6dcc1dba885ad64cbfb28e9f0"},
+    {file = "scipy-1.6.1-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:5da5471aed911fe7e52b86bf9ea32fb55ae93e2f0fac66c32e58897cfb02fa07"},
+    {file = "scipy-1.6.1-cp39-cp39-win32.whl", hash = "sha256:8e403a337749ed40af60e537cc4d4c03febddcc56cd26e774c9b1b600a70d3e4"},
+    {file = "scipy-1.6.1-cp39-cp39-win_amd64.whl", hash = "sha256:a5193a098ae9f29af283dcf0041f762601faf2e595c0db1da929875b7570353f"},
+    {file = "scipy-1.6.1.tar.gz", hash = "sha256:c4fceb864890b6168e79b0e714c585dbe2fd4222768ee90bc1aa0f8218691b11"},
 ]
 six = [
-    {file = "six-1.15.0-py2.py3-none-any.whl", hash = "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced"},
-    {file = "six-1.15.0.tar.gz", hash = "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259"},
+    {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"},
+    {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"},
+]
+typing-extensions = [
+    {file = "typing_extensions-3.10.0.0-py2-none-any.whl", hash = "sha256:0ac0f89795dd19de6b97debb0c6af1c70987fd80a2d62d1958f7e56fcc31b497"},
+    {file = "typing_extensions-3.10.0.0-py3-none-any.whl", hash = "sha256:779383f6086d90c99ae41cf0ff39aac8a7937a9283ce0a414e5dd782f4c94a84"},
+    {file = "typing_extensions-3.10.0.0.tar.gz", hash = "sha256:50b6f157849174217d0656f99dc82fe932884fb250826c18350e159ec6cdf342"},
 ]
 zipp = [
-    {file = "zipp-3.3.0-py3-none-any.whl", hash = "sha256:eed8ec0b8d1416b2ca33516a37a08892442f3954dee131e92cfd92d8fe3e7066"},
-    {file = "zipp-3.3.0.tar.gz", hash = "sha256:64ad89efee774d1897a58607895d80789c59778ea02185dd846ac38394a8642b"},
+    {file = "zipp-3.4.1-py3-none-any.whl", hash = "sha256:51cb66cc54621609dd593d1787f286ee42a5c0adbb4b29abea5a63edc3e03098"},
+    {file = "zipp-3.4.1.tar.gz", hash = "sha256:3607921face881ba3e026887d8150cca609d517579abe052ac81fc5aeffdbd76"},
 ]
diff --git a/pyproject.toml b/pyproject.toml
index b56fdbd..ea84a8e 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -10,7 +10,7 @@ repository = "https://github.com/kngwyu/mujoco-maze"
 homepage = "https://github.com/kngwyu/mujoco-maze"
 
 [tool.poetry.dependencies]
-python = ">=3.6.1"  # Compatible python versions must be declared here
+python = ">=3.7"  # Compatible python versions must be declared here
 gym = ">=0.16"
 mujoco-py = ">=1.5"
 

From 5161db917174884b23fd334ab1c470bc83b10795 Mon Sep 17 00:00:00 2001
From: kngwyu <yuji.kngw.80s.revive@gmail.com>
Date: Thu, 3 Jun 2021 20:29:54 +0900
Subject: [PATCH 23/24] Dont use numpy.complex not to suppress warnings

---
 mujoco_maze/maze_env_utils.py | 20 ++++-----
 poetry.lock                   | 80 +++++++++++++++++++++++++++--------
 pyproject.toml                |  2 +-
 tests/test_intersect.py       |  2 +-
 4 files changed, 74 insertions(+), 30 deletions(-)

diff --git a/mujoco_maze/maze_env_utils.py b/mujoco_maze/maze_env_utils.py
index f9edfe1..21f35d1 100644
--- a/mujoco_maze/maze_env_utils.py
+++ b/mujoco_maze/maze_env_utils.py
@@ -13,7 +13,7 @@ from typing import Any, List, Optional, Sequence, Tuple, Union
 import numpy as np
 
 Self = Any
-Point = np.complex
+Point = complex
 
 
 class MazeCell(Enum):
@@ -87,11 +87,11 @@ class Line:
         p1: Union[Sequence[float], Point],
         p2: Union[Sequence[float], Point],
     ) -> None:
-        self.p1 = p1 if isinstance(p1, Point) else np.complex(*p1)
-        self.p2 = p2 if isinstance(p2, Point) else np.complex(*p2)
+        self.p1 = p1 if isinstance(p1, Point) else complex(*p1)
+        self.p2 = p2 if isinstance(p2, Point) else complex(*p2)
         self.v1 = self.p2 - self.p1
-        self.conj_v1 = np.conjugate(self.v1)
-        self.norm = np.absolute(self.v1)
+        self.conj_v1 = self.v1.conjugate()
+        self.norm = abs(self.v1)
 
     def _intersect(self, other: Self) -> bool:
         v2 = other.p1 - self.p1
@@ -100,15 +100,15 @@ class Line:
 
     def _projection(self, p: Point) -> Point:
         nv1 = -self.v1
-        nv1_norm = np.absolute(nv1) ** 2
-        scale = np.real(np.conjugate(p - self.p1) * nv1) / nv1_norm
+        nv1_norm = abs(nv1) ** 2
+        scale = ((p - self.p1).conjugate() * nv1).real / nv1_norm
         return self.p1 + nv1 * scale
 
     def reflection(self, p: Point) -> Point:
         return p + 2.0 * (self._projection(p) - p)
 
     def distance(self, p: Point) -> float:
-        return np.absolute(p - self._projection(p))
+        return abs(p - self._projection(p))
 
     def intersect(self, other: Self) -> Point:
         if self._intersect(other) and other._intersect(self):
@@ -198,9 +198,9 @@ class CollisionDetector:
         if len(collisions) == 0:
             return None
         col = collisions[0]
-        dist = np.absolute(col._point - move.p1)
+        dist = abs(col._point - move.p1)
         for collision in collisions[1:]:
-            new_dist = np.absolute(collision._point - move.p1)
+            new_dist = abs(collision._point - move.p1)
             if new_dist < dist:
                 col, dist = collision, new_dist
         return col
diff --git a/poetry.lock b/poetry.lock
index 832a57f..68d0bf0 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -133,12 +133,12 @@ docs = ["sphinx", "jaraco.packaging (>=8.2)", "rst.linker (>=1.9)"]
 testing = ["pytest (>=4.6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.0.1)", "packaging", "pep517", "pyfakefs", "flufl.flake8", "pytest-black (>=0.3.7)", "pytest-mypy", "importlib-resources (>=1.3)"]
 
 [[package]]
-name = "more-itertools"
-version = "8.8.0"
-description = "More routines for operating on iterables, beyond itertools"
+name = "iniconfig"
+version = "1.1.1"
+description = "iniconfig: brain-dead simple config-ini parsing"
 category = "dev"
 optional = false
-python-versions = ">=3.5"
+python-versions = "*"
 
 [[package]]
 name = "mujoco-py"
@@ -164,6 +164,17 @@ category = "main"
 optional = false
 python-versions = ">=3.7"
 
+[[package]]
+name = "packaging"
+version = "20.9"
+description = "Core utilities for Python packages"
+category = "dev"
+optional = false
+python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
+
+[package.dependencies]
+pyparsing = ">=2.0.2"
+
 [[package]]
 name = "pillow"
 version = "8.2.0"
@@ -210,22 +221,35 @@ category = "main"
 optional = false
 python-versions = "*"
 
+[[package]]
+name = "pyparsing"
+version = "2.4.7"
+description = "Python parsing module"
+category = "dev"
+optional = false
+python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*"
+
 [[package]]
 name = "pytest"
-version = "3.10.1"
+version = "6.2.4"
 description = "pytest: simple powerful testing with Python"
 category = "dev"
 optional = false
-python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
+python-versions = ">=3.6"
 
 [package.dependencies]
-atomicwrites = ">=1.0"
-attrs = ">=17.4.0"
+atomicwrites = {version = ">=1.0", markers = "sys_platform == \"win32\""}
+attrs = ">=19.2.0"
 colorama = {version = "*", markers = "sys_platform == \"win32\""}
-more-itertools = ">=4.0.0"
-pluggy = ">=0.7"
-py = ">=1.5.0"
-six = ">=1.10.0"
+importlib-metadata = {version = ">=0.12", markers = "python_version < \"3.8\""}
+iniconfig = "*"
+packaging = "*"
+pluggy = ">=0.12,<1.0.0a1"
+py = ">=1.8.2"
+toml = "*"
+
+[package.extras]
+testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"]
 
 [[package]]
 name = "scipy"
@@ -246,6 +270,14 @@ category = "main"
 optional = false
 python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*"
 
+[[package]]
+name = "toml"
+version = "0.10.2"
+description = "Python Library for Tom's Obvious, Minimal Language"
+category = "dev"
+optional = false
+python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*"
+
 [[package]]
 name = "typing-extensions"
 version = "3.10.0.0"
@@ -269,7 +301,7 @@ testing = ["pytest (>=4.6)", "pytest-checkdocs (>=1.2.3)", "pytest-flake8", "pyt
 [metadata]
 lock-version = "1.1"
 python-versions = ">=3.7"  # Compatible python versions must be declared here
-content-hash = "9234f9b51f7c3fab1f9eefcf7f08e01355ce357941e935359e3145e95fb676f3"
+content-hash = "0bbe340523b524544248f4376878056ed5f0f57ec672a62c007543474fb83d05"
 
 [metadata.files]
 atomicwrites = [
@@ -400,9 +432,9 @@ importlib-metadata = [
     {file = "importlib_metadata-4.4.0-py3-none-any.whl", hash = "sha256:960d52ba7c21377c990412aca380bf3642d734c2eaab78a2c39319f67c6a5786"},
     {file = "importlib_metadata-4.4.0.tar.gz", hash = "sha256:e592faad8de1bda9fe920cf41e15261e7131bcf266c30306eec00e8e225c1dd5"},
 ]
-more-itertools = [
-    {file = "more-itertools-8.8.0.tar.gz", hash = "sha256:83f0308e05477c68f56ea3a888172c78ed5d5b3c282addb67508e7ba6c8f813a"},
-    {file = "more_itertools-8.8.0-py3-none-any.whl", hash = "sha256:2cf89ec599962f2ddc4d568a05defc40e0a587fbc10d5989713638864c36be4d"},
+iniconfig = [
+    {file = "iniconfig-1.1.1-py2.py3-none-any.whl", hash = "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3"},
+    {file = "iniconfig-1.1.1.tar.gz", hash = "sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32"},
 ]
 mujoco-py = [
     {file = "mujoco-py-2.0.2.13.tar.gz", hash = "sha256:d6ae66276b565af9063597fda70683a89c7356290f5ac3961b794ee90ec50eea"},
@@ -433,6 +465,10 @@ numpy = [
     {file = "numpy-1.20.3-pp37-pypy37_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:4e465afc3b96dbc80cf4a5273e5e2b1e3451286361b4af70ce1adb2984d392f9"},
     {file = "numpy-1.20.3.zip", hash = "sha256:e55185e51b18d788e49fe8305fd73ef4470596b33fc2c1ceb304566b99c71a69"},
 ]
+packaging = [
+    {file = "packaging-20.9-py2.py3-none-any.whl", hash = "sha256:67714da7f7bc052e064859c05c595155bd1ee9f69f76557e21f051443c20947a"},
+    {file = "packaging-20.9.tar.gz", hash = "sha256:5b327ac1320dc863dca72f4514ecc086f31186744b84a230374cc1fd776feae5"},
+]
 pillow = [
     {file = "Pillow-8.2.0-cp36-cp36m-macosx_10_10_x86_64.whl", hash = "sha256:dc38f57d8f20f06dd7c3161c59ca2c86893632623f33a42d592f097b00f720a9"},
     {file = "Pillow-8.2.0-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:a013cbe25d20c2e0c4e85a9daf438f85121a4d0344ddc76e33fd7e3965d9af4b"},
@@ -485,9 +521,13 @@ pyglet = [
     {file = "pyglet-1.5.15-py3-none-any.whl", hash = "sha256:4401cc176580e4e17e2df8bbf7536f27e691327dc3f38f209a12f1859c70aed2"},
     {file = "pyglet-1.5.15.zip", hash = "sha256:da9d8337388cedabf1f1c5dc21a45bb2b0e5327fba47f996c8573818c3dfa478"},
 ]
+pyparsing = [
+    {file = "pyparsing-2.4.7-py2.py3-none-any.whl", hash = "sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b"},
+    {file = "pyparsing-2.4.7.tar.gz", hash = "sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1"},
+]
 pytest = [
-    {file = "pytest-3.10.1-py2.py3-none-any.whl", hash = "sha256:3f193df1cfe1d1609d4c583838bea3d532b18d6160fd3f55c9447fdca30848ec"},
-    {file = "pytest-3.10.1.tar.gz", hash = "sha256:e246cf173c01169b9617fc07264b7b1316e78d7a650055235d6d897bc80d9660"},
+    {file = "pytest-6.2.4-py3-none-any.whl", hash = "sha256:91ef2131a9bd6be8f76f1f08eac5c5317221d6ad1e143ae03894b862e8976890"},
+    {file = "pytest-6.2.4.tar.gz", hash = "sha256:50bcad0a0b9c5a72c8e4e7c9855a3ad496ca6a881a3641b4260605450772c54b"},
 ]
 scipy = [
     {file = "scipy-1.6.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a15a1f3fc0abff33e792d6049161b7795909b40b97c6cc2934ed54384017ab76"},
@@ -514,6 +554,10 @@ six = [
     {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"},
     {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"},
 ]
+toml = [
+    {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"},
+    {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"},
+]
 typing-extensions = [
     {file = "typing_extensions-3.10.0.0-py2-none-any.whl", hash = "sha256:0ac0f89795dd19de6b97debb0c6af1c70987fd80a2d62d1958f7e56fcc31b497"},
     {file = "typing_extensions-3.10.0.0-py3-none-any.whl", hash = "sha256:779383f6086d90c99ae41cf0ff39aac8a7937a9283ce0a414e5dd782f4c94a84"},
diff --git a/pyproject.toml b/pyproject.toml
index ea84a8e..d76849c 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -15,7 +15,7 @@ gym = ">=0.16"
 mujoco-py = ">=1.5"
 
 [tool.poetry.dev-dependencies]
-pytest = "^3.0"
+pytest = "^6.2"
 
 [tool.poetry.scripts]
 test = "pytest:main"
diff --git a/tests/test_intersect.py b/tests/test_intersect.py
index 2378126..3342f41 100644
--- a/tests/test_intersect.py
+++ b/tests/test_intersect.py
@@ -13,7 +13,7 @@ from mujoco_maze.maze_env_utils import Line
 )
 def test_distance(l1, l2, p, ans):
     line = Line(l1, l2)
-    point = np.complex(*p)
+    point = complex(*p)
     assert abs(line.distance(point) - ans) <= 1e-8
 
 

From 620ca1cb0f36a99741864a01b6903f246b5594e8 Mon Sep 17 00:00:00 2001
From: kngwyu <yuji.kngw.80s.revive@gmail.com>
Date: Thu, 3 Jun 2021 21:31:44 +0900
Subject: [PATCH 24/24] Allow Python 3.6 again

---
 .github/workflows/main.yml |   2 +-
 poetry.lock                | 116 +++++++++++++++++++++----------------
 pyproject.toml             |   2 +-
 3 files changed, 68 insertions(+), 52 deletions(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index d8887a8..4e33292 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -14,7 +14,7 @@ jobs:
   ci:
     strategy:
       matrix:
-        python-version: [3.7, 3.8]
+        python-version: [3.6, 3.7, 3.8]
     runs-on: ubuntu-18.04
     env:
       LD_LIBRARY_PATH: /home/runner/.mujoco/mujoco200/bin
diff --git a/poetry.lock b/poetry.lock
index 68d0bf0..0e4800d 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -158,11 +158,11 @@ numpy = ">=1.11"
 
 [[package]]
 name = "numpy"
-version = "1.20.3"
+version = "1.19.5"
 description = "NumPy is the fundamental package for array computing with Python."
 category = "main"
 optional = false
-python-versions = ">=3.7"
+python-versions = ">=3.6"
 
 [[package]]
 name = "packaging"
@@ -253,14 +253,14 @@ testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xm
 
 [[package]]
 name = "scipy"
-version = "1.6.1"
+version = "1.5.4"
 description = "SciPy: Scientific Library for Python"
 category = "main"
 optional = false
-python-versions = ">=3.7"
+python-versions = ">=3.6"
 
 [package.dependencies]
-numpy = ">=1.16.5"
+numpy = ">=1.14.5"
 
 [[package]]
 name = "six"
@@ -300,8 +300,8 @@ testing = ["pytest (>=4.6)", "pytest-checkdocs (>=1.2.3)", "pytest-flake8", "pyt
 
 [metadata]
 lock-version = "1.1"
-python-versions = ">=3.7"  # Compatible python versions must be declared here
-content-hash = "0bbe340523b524544248f4376878056ed5f0f57ec672a62c007543474fb83d05"
+python-versions = ">=3.6"  # Compatible python versions must be declared here
+content-hash = "7ee84c79fcf3f786abecf596c63f654e3b635b34a139b01835b0e503983fdbd5"
 
 [metadata.files]
 atomicwrites = [
@@ -440,30 +440,40 @@ mujoco-py = [
     {file = "mujoco-py-2.0.2.13.tar.gz", hash = "sha256:d6ae66276b565af9063597fda70683a89c7356290f5ac3961b794ee90ec50eea"},
 ]
 numpy = [
-    {file = "numpy-1.20.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:70eb5808127284c4e5c9e836208e09d685a7978b6a216db85960b1a112eeace8"},
-    {file = "numpy-1.20.3-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:6ca2b85a5997dabc38301a22ee43c82adcb53ff660b89ee88dded6b33687e1d8"},
-    {file = "numpy-1.20.3-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:c5bf0e132acf7557fc9bb8ded8b53bbbbea8892f3c9a1738205878ca9434206a"},
-    {file = "numpy-1.20.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:db250fd3e90117e0312b611574cd1b3f78bec046783195075cbd7ba9c3d73f16"},
-    {file = "numpy-1.20.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:637d827248f447e63585ca3f4a7d2dfaa882e094df6cfa177cc9cf9cd6cdf6d2"},
-    {file = "numpy-1.20.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:8b7bb4b9280da3b2856cb1fc425932f46fba609819ee1c62256f61799e6a51d2"},
-    {file = "numpy-1.20.3-cp37-cp37m-win32.whl", hash = "sha256:67d44acb72c31a97a3d5d33d103ab06d8ac20770e1c5ad81bdb3f0c086a56cf6"},
-    {file = "numpy-1.20.3-cp37-cp37m-win_amd64.whl", hash = "sha256:43909c8bb289c382170e0282158a38cf306a8ad2ff6dfadc447e90f9961bef43"},
-    {file = "numpy-1.20.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:f1452578d0516283c87608a5a5548b0cdde15b99650efdfd85182102ef7a7c17"},
-    {file = "numpy-1.20.3-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:6e51534e78d14b4a009a062641f465cfaba4fdcb046c3ac0b1f61dd97c861b1b"},
-    {file = "numpy-1.20.3-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:e515c9a93aebe27166ec9593411c58494fa98e5fcc219e47260d9ab8a1cc7f9f"},
-    {file = "numpy-1.20.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c1c09247ccea742525bdb5f4b5ceeacb34f95731647fe55774aa36557dbb5fa4"},
-    {file = "numpy-1.20.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:66fbc6fed94a13b9801fb70b96ff30605ab0a123e775a5e7a26938b717c5d71a"},
-    {file = "numpy-1.20.3-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:ea9cff01e75a956dbee133fa8e5b68f2f92175233de2f88de3a682dd94deda65"},
-    {file = "numpy-1.20.3-cp38-cp38-win32.whl", hash = "sha256:f39a995e47cb8649673cfa0579fbdd1cdd33ea497d1728a6cb194d6252268e48"},
-    {file = "numpy-1.20.3-cp38-cp38-win_amd64.whl", hash = "sha256:1676b0a292dd3c99e49305a16d7a9f42a4ab60ec522eac0d3dd20cdf362ac010"},
-    {file = "numpy-1.20.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:830b044f4e64a76ba71448fce6e604c0fc47a0e54d8f6467be23749ac2cbd2fb"},
-    {file = "numpy-1.20.3-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:55b745fca0a5ab738647d0e4db099bd0a23279c32b31a783ad2ccea729e632df"},
-    {file = "numpy-1.20.3-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:5d050e1e4bc9ddb8656d7b4f414557720ddcca23a5b88dd7cff65e847864c400"},
-    {file = "numpy-1.20.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a9c65473ebc342715cb2d7926ff1e202c26376c0dcaaee85a1fd4b8d8c1d3b2f"},
-    {file = "numpy-1.20.3-cp39-cp39-win32.whl", hash = "sha256:16f221035e8bd19b9dc9a57159e38d2dd060b48e93e1d843c49cb370b0f415fd"},
-    {file = "numpy-1.20.3-cp39-cp39-win_amd64.whl", hash = "sha256:6690080810f77485667bfbff4f69d717c3be25e5b11bb2073e76bb3f578d99b4"},
-    {file = "numpy-1.20.3-pp37-pypy37_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:4e465afc3b96dbc80cf4a5273e5e2b1e3451286361b4af70ce1adb2984d392f9"},
-    {file = "numpy-1.20.3.zip", hash = "sha256:e55185e51b18d788e49fe8305fd73ef4470596b33fc2c1ceb304566b99c71a69"},
+    {file = "numpy-1.19.5-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:cc6bd4fd593cb261332568485e20a0712883cf631f6f5e8e86a52caa8b2b50ff"},
+    {file = "numpy-1.19.5-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:aeb9ed923be74e659984e321f609b9ba54a48354bfd168d21a2b072ed1e833ea"},
+    {file = "numpy-1.19.5-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:8b5e972b43c8fc27d56550b4120fe6257fdc15f9301914380b27f74856299fea"},
+    {file = "numpy-1.19.5-cp36-cp36m-manylinux2010_i686.whl", hash = "sha256:43d4c81d5ffdff6bae58d66a3cd7f54a7acd9a0e7b18d97abb255defc09e3140"},
+    {file = "numpy-1.19.5-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:a4646724fba402aa7504cd48b4b50e783296b5e10a524c7a6da62e4a8ac9698d"},
+    {file = "numpy-1.19.5-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:2e55195bc1c6b705bfd8ad6f288b38b11b1af32f3c8289d6c50d47f950c12e76"},
+    {file = "numpy-1.19.5-cp36-cp36m-win32.whl", hash = "sha256:39b70c19ec771805081578cc936bbe95336798b7edf4732ed102e7a43ec5c07a"},
+    {file = "numpy-1.19.5-cp36-cp36m-win_amd64.whl", hash = "sha256:dbd18bcf4889b720ba13a27ec2f2aac1981bd41203b3a3b27ba7a33f88ae4827"},
+    {file = "numpy-1.19.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:603aa0706be710eea8884af807b1b3bc9fb2e49b9f4da439e76000f3b3c6ff0f"},
+    {file = "numpy-1.19.5-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:cae865b1cae1ec2663d8ea56ef6ff185bad091a5e33ebbadd98de2cfa3fa668f"},
+    {file = "numpy-1.19.5-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:36674959eed6957e61f11c912f71e78857a8d0604171dfd9ce9ad5cbf41c511c"},
+    {file = "numpy-1.19.5-cp37-cp37m-manylinux2010_i686.whl", hash = "sha256:06fab248a088e439402141ea04f0fffb203723148f6ee791e9c75b3e9e82f080"},
+    {file = "numpy-1.19.5-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:6149a185cece5ee78d1d196938b2a8f9d09f5a5ebfbba66969302a778d5ddd1d"},
+    {file = "numpy-1.19.5-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:50a4a0ad0111cc1b71fa32dedd05fa239f7fb5a43a40663269bb5dc7877cfd28"},
+    {file = "numpy-1.19.5-cp37-cp37m-win32.whl", hash = "sha256:d051ec1c64b85ecc69531e1137bb9751c6830772ee5c1c426dbcfe98ef5788d7"},
+    {file = "numpy-1.19.5-cp37-cp37m-win_amd64.whl", hash = "sha256:a12ff4c8ddfee61f90a1633a4c4afd3f7bcb32b11c52026c92a12e1325922d0d"},
+    {file = "numpy-1.19.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:cf2402002d3d9f91c8b01e66fbb436a4ed01c6498fffed0e4c7566da1d40ee1e"},
+    {file = "numpy-1.19.5-cp38-cp38-manylinux1_i686.whl", hash = "sha256:1ded4fce9cfaaf24e7a0ab51b7a87be9038ea1ace7f34b841fe3b6894c721d1c"},
+    {file = "numpy-1.19.5-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:012426a41bc9ab63bb158635aecccc7610e3eff5d31d1eb43bc099debc979d94"},
+    {file = "numpy-1.19.5-cp38-cp38-manylinux2010_i686.whl", hash = "sha256:759e4095edc3c1b3ac031f34d9459fa781777a93ccc633a472a5468587a190ff"},
+    {file = "numpy-1.19.5-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:a9d17f2be3b427fbb2bce61e596cf555d6f8a56c222bd2ca148baeeb5e5c783c"},
+    {file = "numpy-1.19.5-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:99abf4f353c3d1a0c7a5f27699482c987cf663b1eac20db59b8c7b061eabd7fc"},
+    {file = "numpy-1.19.5-cp38-cp38-win32.whl", hash = "sha256:384ec0463d1c2671170901994aeb6dce126de0a95ccc3976c43b0038a37329c2"},
+    {file = "numpy-1.19.5-cp38-cp38-win_amd64.whl", hash = "sha256:811daee36a58dc79cf3d8bdd4a490e4277d0e4b7d103a001a4e73ddb48e7e6aa"},
+    {file = "numpy-1.19.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c843b3f50d1ab7361ca4f0b3639bf691569493a56808a0b0c54a051d260b7dbd"},
+    {file = "numpy-1.19.5-cp39-cp39-manylinux1_i686.whl", hash = "sha256:d6631f2e867676b13026e2846180e2c13c1e11289d67da08d71cacb2cd93d4aa"},
+    {file = "numpy-1.19.5-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:7fb43004bce0ca31d8f13a6eb5e943fa73371381e53f7074ed21a4cb786c32f8"},
+    {file = "numpy-1.19.5-cp39-cp39-manylinux2010_i686.whl", hash = "sha256:2ea52bd92ab9f768cc64a4c3ef8f4b2580a17af0a5436f6126b08efbd1838371"},
+    {file = "numpy-1.19.5-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:400580cbd3cff6ffa6293df2278c75aef2d58d8d93d3c5614cd67981dae68ceb"},
+    {file = "numpy-1.19.5-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:df609c82f18c5b9f6cb97271f03315ff0dbe481a2a02e56aeb1b1a985ce38e60"},
+    {file = "numpy-1.19.5-cp39-cp39-win32.whl", hash = "sha256:ab83f24d5c52d60dbc8cd0528759532736b56db58adaa7b5f1f76ad551416a1e"},
+    {file = "numpy-1.19.5-cp39-cp39-win_amd64.whl", hash = "sha256:0eef32ca3132a48e43f6a0f5a82cb508f22ce5a3d6f67a8329c81c8e226d3f6e"},
+    {file = "numpy-1.19.5-pp36-pypy36_pp73-manylinux2010_x86_64.whl", hash = "sha256:a0d53e51a6cb6f0d9082decb7a4cb6dfb33055308c4c44f53103c073f649af73"},
+    {file = "numpy-1.19.5.zip", hash = "sha256:a76f502430dd98d7546e1ea2250a7360c065a5fdea52b2dffe8ae7180909b6f4"},
 ]
 packaging = [
     {file = "packaging-20.9-py2.py3-none-any.whl", hash = "sha256:67714da7f7bc052e064859c05c595155bd1ee9f69f76557e21f051443c20947a"},
@@ -530,25 +540,31 @@ pytest = [
     {file = "pytest-6.2.4.tar.gz", hash = "sha256:50bcad0a0b9c5a72c8e4e7c9855a3ad496ca6a881a3641b4260605450772c54b"},
 ]
 scipy = [
-    {file = "scipy-1.6.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a15a1f3fc0abff33e792d6049161b7795909b40b97c6cc2934ed54384017ab76"},
-    {file = "scipy-1.6.1-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:e79570979ccdc3d165456dd62041d9556fb9733b86b4b6d818af7a0afc15f092"},
-    {file = "scipy-1.6.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:a423533c55fec61456dedee7b6ee7dce0bb6bfa395424ea374d25afa262be261"},
-    {file = "scipy-1.6.1-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:33d6b7df40d197bdd3049d64e8e680227151673465e5d85723b3b8f6b15a6ced"},
-    {file = "scipy-1.6.1-cp37-cp37m-win32.whl", hash = "sha256:6725e3fbb47da428794f243864f2297462e9ee448297c93ed1dcbc44335feb78"},
-    {file = "scipy-1.6.1-cp37-cp37m-win_amd64.whl", hash = "sha256:5fa9c6530b1661f1370bcd332a1e62ca7881785cc0f80c0d559b636567fab63c"},
-    {file = "scipy-1.6.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:bd50daf727f7c195e26f27467c85ce653d41df4358a25b32434a50d8870fc519"},
-    {file = "scipy-1.6.1-cp38-cp38-manylinux1_i686.whl", hash = "sha256:f46dd15335e8a320b0fb4685f58b7471702234cba8bb3442b69a3e1dc329c345"},
-    {file = "scipy-1.6.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:0e5b0ccf63155d90da576edd2768b66fb276446c371b73841e3503be1d63fb5d"},
-    {file = "scipy-1.6.1-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:2481efbb3740977e3c831edfd0bd9867be26387cacf24eb5e366a6a374d3d00d"},
-    {file = "scipy-1.6.1-cp38-cp38-win32.whl", hash = "sha256:68cb4c424112cd4be886b4d979c5497fba190714085f46b8ae67a5e4416c32b4"},
-    {file = "scipy-1.6.1-cp38-cp38-win_amd64.whl", hash = "sha256:5f331eeed0297232d2e6eea51b54e8278ed8bb10b099f69c44e2558c090d06bf"},
-    {file = "scipy-1.6.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:0c8a51d33556bf70367452d4d601d1742c0e806cd0194785914daf19775f0e67"},
-    {file = "scipy-1.6.1-cp39-cp39-manylinux1_i686.whl", hash = "sha256:83bf7c16245c15bc58ee76c5418e46ea1811edcc2e2b03041b804e46084ab627"},
-    {file = "scipy-1.6.1-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:794e768cc5f779736593046c9714e0f3a5940bc6dcc1dba885ad64cbfb28e9f0"},
-    {file = "scipy-1.6.1-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:5da5471aed911fe7e52b86bf9ea32fb55ae93e2f0fac66c32e58897cfb02fa07"},
-    {file = "scipy-1.6.1-cp39-cp39-win32.whl", hash = "sha256:8e403a337749ed40af60e537cc4d4c03febddcc56cd26e774c9b1b600a70d3e4"},
-    {file = "scipy-1.6.1-cp39-cp39-win_amd64.whl", hash = "sha256:a5193a098ae9f29af283dcf0041f762601faf2e595c0db1da929875b7570353f"},
-    {file = "scipy-1.6.1.tar.gz", hash = "sha256:c4fceb864890b6168e79b0e714c585dbe2fd4222768ee90bc1aa0f8218691b11"},
+    {file = "scipy-1.5.4-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:4f12d13ffbc16e988fa40809cbbd7a8b45bc05ff6ea0ba8e3e41f6f4db3a9e47"},
+    {file = "scipy-1.5.4-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:a254b98dbcc744c723a838c03b74a8a34c0558c9ac5c86d5561703362231107d"},
+    {file = "scipy-1.5.4-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:368c0f69f93186309e1b4beb8e26d51dd6f5010b79264c0f1e9ca00cd92ea8c9"},
+    {file = "scipy-1.5.4-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:4598cf03136067000855d6b44d7a1f4f46994164bcd450fb2c3d481afc25dd06"},
+    {file = "scipy-1.5.4-cp36-cp36m-win32.whl", hash = "sha256:e98d49a5717369d8241d6cf33ecb0ca72deee392414118198a8e5b4c35c56340"},
+    {file = "scipy-1.5.4-cp36-cp36m-win_amd64.whl", hash = "sha256:65923bc3809524e46fb7eb4d6346552cbb6a1ffc41be748535aa502a2e3d3389"},
+    {file = "scipy-1.5.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:9ad4fcddcbf5dc67619379782e6aeef41218a79e17979aaed01ed099876c0e62"},
+    {file = "scipy-1.5.4-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:f87b39f4d69cf7d7529d7b1098cb712033b17ea7714aed831b95628f483fd012"},
+    {file = "scipy-1.5.4-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:25b241034215247481f53355e05f9e25462682b13bd9191359075682adcd9554"},
+    {file = "scipy-1.5.4-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:fa789583fc94a7689b45834453fec095245c7e69c58561dc159b5d5277057e4c"},
+    {file = "scipy-1.5.4-cp37-cp37m-win32.whl", hash = "sha256:d6d25c41a009e3c6b7e757338948d0076ee1dd1770d1c09ec131f11946883c54"},
+    {file = "scipy-1.5.4-cp37-cp37m-win_amd64.whl", hash = "sha256:2c872de0c69ed20fb1a9b9cf6f77298b04a26f0b8720a5457be08be254366c6e"},
+    {file = "scipy-1.5.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:e360cb2299028d0b0d0f65a5c5e51fc16a335f1603aa2357c25766c8dab56938"},
+    {file = "scipy-1.5.4-cp38-cp38-manylinux1_i686.whl", hash = "sha256:3397c129b479846d7eaa18f999369a24322d008fac0782e7828fa567358c36ce"},
+    {file = "scipy-1.5.4-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:168c45c0c32e23f613db7c9e4e780bc61982d71dcd406ead746c7c7c2f2004ce"},
+    {file = "scipy-1.5.4-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:213bc59191da2f479984ad4ec39406bf949a99aba70e9237b916ce7547b6ef42"},
+    {file = "scipy-1.5.4-cp38-cp38-win32.whl", hash = "sha256:634568a3018bc16a83cda28d4f7aed0d803dd5618facb36e977e53b2df868443"},
+    {file = "scipy-1.5.4-cp38-cp38-win_amd64.whl", hash = "sha256:b03c4338d6d3d299e8ca494194c0ae4f611548da59e3c038813f1a43976cb437"},
+    {file = "scipy-1.5.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:3d5db5d815370c28d938cf9b0809dade4acf7aba57eaf7ef733bfedc9b2474c4"},
+    {file = "scipy-1.5.4-cp39-cp39-manylinux1_i686.whl", hash = "sha256:6b0ceb23560f46dd236a8ad4378fc40bad1783e997604ba845e131d6c680963e"},
+    {file = "scipy-1.5.4-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:ed572470af2438b526ea574ff8f05e7f39b44ac37f712105e57fc4d53a6fb660"},
+    {file = "scipy-1.5.4-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:8c8d6ca19c8497344b810b0b0344f8375af5f6bb9c98bd42e33f747417ab3f57"},
+    {file = "scipy-1.5.4-cp39-cp39-win32.whl", hash = "sha256:d84cadd7d7998433334c99fa55bcba0d8b4aeff0edb123b2a1dfcface538e474"},
+    {file = "scipy-1.5.4-cp39-cp39-win_amd64.whl", hash = "sha256:cc1f78ebc982cd0602c9a7615d878396bec94908db67d4ecddca864d049112f2"},
+    {file = "scipy-1.5.4.tar.gz", hash = "sha256:4a453d5e5689de62e5d38edf40af3f17560bfd63c9c5bd228c18c1f99afa155b"},
 ]
 six = [
     {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"},
diff --git a/pyproject.toml b/pyproject.toml
index d76849c..bb469f0 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -10,7 +10,7 @@ repository = "https://github.com/kngwyu/mujoco-maze"
 homepage = "https://github.com/kngwyu/mujoco-maze"
 
 [tool.poetry.dependencies]
-python = ">=3.7"  # Compatible python versions must be declared here
+python = ">=3.6"  # Compatible python versions must be declared here
 gym = ">=0.16"
 mujoco-py = ">=1.5"