Add MuJoCo compilation debugging and continue validation tests

- Add robomimic square test (continuing pre-training validation)
- Create MuJoCo environment fix scripts for debugging compilation
- Update experiment plan with latest test results
- Robomimic can pre-training validated successfully

slurm/dev_tests/test_hopper_finetune_v2.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+#SBATCH --job-name=dppo_hop_ft_v2
+#SBATCH --account=hk-project-p0022232
+#SBATCH --partition=dev_accelerated
+#SBATCH --gres=gpu:1
+#SBATCH --nodes=1
+#SBATCH --ntasks-per-node=1
+#SBATCH --cpus-per-task=8
+#SBATCH --time=00:30:00
+#SBATCH --mem=24G
+#SBATCH --output=logs/dppo_hop_ft_v2_%j.out
+#SBATCH --error=logs/dppo_hop_ft_v2_%j.err
+
+module load devel/cuda/12.4
+
+# MuJoCo environment for fine-tuning with compilation fixes
+export MUJOCO_PY_MUJOCO_PATH=/home/hk-project-robolear/ys1087/.mujoco/mujoco210
+export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/home/hk-project-robolear/ys1087/.mujoco/mujoco210/bin:/usr/lib/nvidia
+export MUJOCO_GL=egl
+
+# Force GCC compilation environment
+export CC=gcc
+export CXX=g++
+export CFLAGS="-w"
+export CXXFLAGS="-w"
+
+export WANDB_MODE=online
+export DPPO_WANDB_ENTITY=${DPPO_WANDB_ENTITY:-"dominik_roth"}
+export DPPO_DATA_DIR=${DPPO_DATA_DIR:-$SLURM_SUBMIT_DIR/data}
+export DPPO_LOG_DIR=${DPPO_LOG_DIR:-$SLURM_SUBMIT_DIR/log}
+
+cd $SLURM_SUBMIT_DIR
+source .venv/bin/activate
+
+echo "Testing hopper finetune v2 with stdio.h fix and cleared cache..."
+python script/run.py --config-name=ft_ppo_diffusion_mlp \
+    --config-dir=cfg/gym/finetune/hopper-v2 \
+    train.n_train_itr=10 \
+    train.save_model_freq=5

slurm/dev_tests/test_mujoco_direct_fix.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+#SBATCH --job-name=dppo_mujoco_fix
+#SBATCH --account=hk-project-p0022232
+#SBATCH --partition=dev_accelerated
+#SBATCH --gres=gpu:1
+#SBATCH --nodes=1
+#SBATCH --ntasks-per-node=1
+#SBATCH --cpus-per-task=8
+#SBATCH --time=00:30:00
+#SBATCH --mem=24G
+#SBATCH --output=logs/dppo_mujoco_fix_%j.out
+#SBATCH --error=logs/dppo_mujoco_fix_%j.err
+
+# Purge all modules to avoid Intel compiler interference
+module purge
+module load devel/cuda/12.4
+
+# Force system GCC
+export PATH="/usr/bin:$PATH"
+export CC=/usr/bin/gcc
+export CXX=/usr/bin/g++
+
+# Clear all Intel-related environment variables
+unset ICC_PATH
+unset INTEL_COMPILER_HOME  
+unset INTEL_LICENSE_FILE
+
+cd $SLURM_SUBMIT_DIR
+source .venv/bin/activate
+
+echo "Testing MuJoCo compilation with pure GCC environment..."
+python fix_mujoco_env.py

slurm/dev_tests/test_robomimic_square.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+#SBATCH --job-name=dppo_robo_square
+#SBATCH --account=hk-project-p0022232
+#SBATCH --partition=dev_accelerated
+#SBATCH --gres=gpu:1
+#SBATCH --nodes=1
+#SBATCH --ntasks-per-node=1
+#SBATCH --cpus-per-task=8
+#SBATCH --time=00:30:00
+#SBATCH --mem=24G
+#SBATCH --output=logs/dppo_robo_square_%j.out
+#SBATCH --error=logs/dppo_robo_square_%j.err
+
+module load devel/cuda/12.4
+export WANDB_MODE=online
+export DPPO_WANDB_ENTITY=${DPPO_WANDB_ENTITY:-"dominik_roth"}
+export DPPO_DATA_DIR=${DPPO_DATA_DIR:-$SLURM_SUBMIT_DIR/data}
+export DPPO_LOG_DIR=${DPPO_LOG_DIR:-$SLURM_SUBMIT_DIR/log}
+
+cd $SLURM_SUBMIT_DIR
+source .venv/bin/activate
+
+echo "Testing Robomimic square pretrain..."
+python script/run.py --config-name=pre_diffusion_mlp \
+    --config-dir=cfg/robomimic/pretrain/square \
+    train.n_epochs=2 \
+    train.save_model_freq=1