v0.5 to main (#10)

* v0.5 (#9)

* update idql configs

* update awr configs

* update dipo configs

* update qsm configs

* update dqm configs

* update project version to 0.5.0

agent/eval/eval_diffusion_agent.py

@@ -36,7 +36,7 @@ class EvalDiffusionAgent(EvalAgent):
         firsts_trajs = np.zeros((self.n_steps + 1, self.n_envs))
         prev_obs_venv = self.reset_env_all(options_venv=options_venv)
         firsts_trajs[0] = 1
-        reward_trajs = np.empty((0, self.n_envs))
+        reward_trajs = np.zeros((self.n_steps, self.n_envs))
 
         # Collect a set of trajectories from env
         for step in range(self.n_steps):
@@ -57,9 +57,13 @@ class EvalDiffusionAgent(EvalAgent):
             action_venv = output_venv[:, : self.act_steps]
 
             # Apply multi-step action
-            obs_venv, reward_venv, done_venv, info_venv = self.venv.step(action_venv)
-            reward_trajs = np.vstack((reward_trajs, reward_venv[None]))
-            firsts_trajs[step + 1] = done_venv
+            obs_venv, reward_venv, terminated_venv, truncated_venv, info_venv = (
+                self.venv.step(action_venv)
+            )
+            reward_trajs[step] = reward_venv
+            firsts_trajs[step + 1] = terminated_venv | truncated_venv
+            
+            # update for next step
             prev_obs_venv = obs_venv
 
         # Summarize episode reward --- this needs to be handled differently depending on whether the environment is reset after each iteration. Only count episodes that finish within the iteration.

agent/eval/eval_diffusion_img_agent.py

@@ -40,7 +40,7 @@ class EvalImgDiffusionAgent(EvalAgent):
         firsts_trajs = np.zeros((self.n_steps + 1, self.n_envs))
         prev_obs_venv = self.reset_env_all(options_venv=options_venv)
         firsts_trajs[0] = 1
-        reward_trajs = np.empty((0, self.n_envs))
+        reward_trajs = np.zeros((self.n_steps, self.n_envs))
 
         # Collect a set of trajectories from env
         for step in range(self.n_steps):
@@ -60,9 +60,13 @@ class EvalImgDiffusionAgent(EvalAgent):
             action_venv = output_venv[:, : self.act_steps]
 
             # Apply multi-step action
-            obs_venv, reward_venv, done_venv, info_venv = self.venv.step(action_venv)
-            reward_trajs = np.vstack((reward_trajs, reward_venv[None]))
-            firsts_trajs[step + 1] = done_venv
+            obs_venv, reward_venv, terminated_venv, truncated_venv, info_venv = (
+                self.venv.step(action_venv)
+            )
+            reward_trajs[step] = reward_venv
+            firsts_trajs[step + 1] = terminated_venv | truncated_venv
+    
+            # update for next step
             prev_obs_venv = obs_venv
 
         # Summarize episode reward --- this needs to be handled differently depending on whether the environment is reset after each iteration. Only count episodes that finish within the iteration.

agent/eval/eval_gaussian_agent.py

@@ -36,7 +36,7 @@ class EvalGaussianAgent(EvalAgent):
         firsts_trajs = np.zeros((self.n_steps + 1, self.n_envs))
         prev_obs_venv = self.reset_env_all(options_venv=options_venv)
         firsts_trajs[0] = 1
-        reward_trajs = np.empty((0, self.n_envs))
+        reward_trajs = np.zeros((self.n_steps, self.n_envs))
 
         # Collect a set of trajectories from env
         for step in range(self.n_steps):
@@ -55,9 +55,13 @@ class EvalGaussianAgent(EvalAgent):
             action_venv = output_venv[:, : self.act_steps]
 
             # Apply multi-step action
-            obs_venv, reward_venv, done_venv, info_venv = self.venv.step(action_venv)
-            reward_trajs = np.vstack((reward_trajs, reward_venv[None]))
-            firsts_trajs[step + 1] = done_venv
+            obs_venv, reward_venv, terminated_venv, truncated_venv, info_venv = (
+                self.venv.step(action_venv)
+            )
+            reward_trajs[step] = reward_venv
+            firsts_trajs[step + 1] = terminated_venv | truncated_venv
+
+            # update for next step
             prev_obs_venv = obs_venv
 
         # Summarize episode reward --- this needs to be handled differently depending on whether the environment is reset after each iteration. Only count episodes that finish within the iteration.

agent/eval/eval_gaussian_img_agent.py

@@ -40,7 +40,7 @@ class EvalImgGaussianAgent(EvalAgent):
         firsts_trajs = np.zeros((self.n_steps + 1, self.n_envs))
         prev_obs_venv = self.reset_env_all(options_venv=options_venv)
         firsts_trajs[0] = 1
-        reward_trajs = np.empty((0, self.n_envs))
+        reward_trajs = np.zeros((self.n_steps, self.n_envs))
 
         # Collect a set of trajectories from env
         for step in range(self.n_steps):
@@ -58,9 +58,13 @@ class EvalImgGaussianAgent(EvalAgent):
             action_venv = output_venv[:, : self.act_steps]
 
             # Apply multi-step action
-            obs_venv, reward_venv, done_venv, info_venv = self.venv.step(action_venv)
-            reward_trajs = np.vstack((reward_trajs, reward_venv[None]))
-            firsts_trajs[step + 1] = done_venv
+            obs_venv, reward_venv, terminated_venv, truncated_venv, info_venv = (
+                self.venv.step(action_venv)
+            )
+            reward_trajs[step] = reward_venv
+            firsts_trajs[step + 1] = terminated_venv | truncated_venv
+
+            # update for next step
             prev_obs_venv = obs_venv
 
         # Summarize episode reward --- this needs to be handled differently depending on whether the environment is reset after each iteration. Only count episodes that finish within the iteration.