after deadline

alr_envs/alr/mujoco/reacher/alr_reacher.py

@@ -8,7 +8,8 @@ import alr_envs.utils.utils as alr_utils
 
 
 class ALRReacherEnv(MujocoEnv, utils.EzPickle):
-    def __init__(self, steps_before_reward=200, n_links=5, balance=False):
+    def __init__(self, steps_before_reward: int = 200, n_links: int = 5, ctrl_cost_weight: int = 1,
+                 balance: bool = False):
         utils.EzPickle.__init__(**locals())
 
         self._steps = 0
@@ -17,6 +18,7 @@ class ALRReacherEnv(MujocoEnv, utils.EzPickle):
 
         self.balance = balance
         self.balance_weight = 1.0
+        self.ctrl_cost_weight = ctrl_cost_weight
 
         self.reward_weight = 1
         if steps_before_reward == 200:
@@ -40,7 +42,7 @@ class ALRReacherEnv(MujocoEnv, utils.EzPickle):
         angular_vel = 0.0
         reward_balance = 0.0
         is_delayed = self.steps_before_reward > 0
-        reward_ctrl = - np.square(a).sum()
+        reward_ctrl = - np.square(a).sum() * self.ctrl_cost_weight
         if self._steps >= self.steps_before_reward:
             vec = self.get_body_com("fingertip") - self.get_body_com("target")
             reward_dist -= self.reward_weight * np.linalg.norm(vec)
@@ -48,9 +50,9 @@ class ALRReacherEnv(MujocoEnv, utils.EzPickle):
                 # avoid giving this penalty for normal step based case
                 # angular_vel -= 10 * np.linalg.norm(self.sim.data.qvel.flat[:self.n_links])
                 angular_vel -= 10 * np.square(self.sim.data.qvel.flat[:self.n_links]).sum()
-        if is_delayed:
-            # Higher control penalty for sparse reward per timestep
-            reward_ctrl *= 10
+        # if is_delayed:
+        #     # Higher control penalty for sparse reward per timestep
+        #     reward_ctrl *= 10
 
         if self.balance:
             reward_balance -= self.balance_weight * np.abs(
@@ -68,35 +70,42 @@ class ALRReacherEnv(MujocoEnv, utils.EzPickle):
     def viewer_setup(self):
         self.viewer.cam.trackbodyid = 0
 
-    def reset_model(self):
-        qpos = self.init_qpos
-        if not hasattr(self, "goal"):
-            self.goal = np.array([-0.25, 0.25])
-            # self.goal = self.init_qpos.copy()[:2] + 0.05
-        qpos[-2:] = self.goal
-        qvel = self.init_qvel
-        qvel[-2:] = 0
-        self.set_state(qpos, qvel)
-        self._steps = 0
-
-        return self._get_obs()
-
     # def reset_model(self):
-    #     qpos = self.init_qpos.copy()
-    #     while True:
-    #         self.goal = self.np_random.uniform(low=-self.n_links / 10, high=self.n_links / 10, size=2)
-    #         # self.goal = self.np_random.uniform(low=0, high=self.n_links / 10, size=2)
-    #         # self.goal = np.random.uniform(low=[-self.n_links / 10, 0], high=[0, self.n_links / 10], size=2)
-    #         if np.linalg.norm(self.goal) < self.n_links / 10:
-    #             break
+    #     qpos = self.init_qpos
+    #     if not hasattr(self, "goal"):
+    #         self.goal = np.array([-0.25, 0.25])
+    #         # self.goal = self.init_qpos.copy()[:2] + 0.05
     #     qpos[-2:] = self.goal
-    #     qvel = self.init_qvel.copy()
+    #     qvel = self.init_qvel
     #     qvel[-2:] = 0
     #     self.set_state(qpos, qvel)
     #     self._steps = 0
     #
     #     return self._get_obs()
 
+    def reset_model(self):
+        qpos = self.init_qpos.copy()
+        while True:
+            # full space
+            # self.goal = self.np_random.uniform(low=-self.n_links / 10, high=self.n_links / 10, size=2)
+            # I Quadrant
+            # self.goal = self.np_random.uniform(low=0, high=self.n_links / 10, size=2)
+            # II Quadrant
+            # self.goal = np.random.uniform(low=[-self.n_links / 10, 0], high=[0, self.n_links / 10], size=2)
+            # II + III Quadrant
+            # self.goal = np.random.uniform(low=-self.n_links / 10, high=[0, self.n_links / 10], size=2)
+            # I + II Quadrant
+            self.goal = np.random.uniform(low=[-self.n_links / 10, 0], high=self.n_links, size=2)
+            if np.linalg.norm(self.goal) < self.n_links / 10:
+                break
+        qpos[-2:] = self.goal
+        qvel = self.init_qvel.copy()
+        qvel[-2:] = 0
+        self.set_state(qpos, qvel)
+        self._steps = 0
+
+        return self._get_obs()
+
     # def reset_model(self):
     #     qpos = self.np_random.uniform(low=-0.1, high=0.1, size=self.model.nq) + self.init_qpos
     #     while True:
@@ -140,4 +149,4 @@ if __name__ == '__main__':
         if d:
             env.reset()
 
-    env.close()
+    env.close()

alr_envs/alr/mujoco/reacher/mp_wrapper.py

@@ -40,4 +40,4 @@ class MPWrapper(MPEnvWrapper):
 
     @property
     def dt(self) -> Union[float, int]:
-        return self.env.dt
+        return self.env.dt