247 lines
23 KiB
HTML
247 lines
23 KiB
HTML
<!DOCTYPE html>
|
|
<html class="writer-html5" lang="en" >
|
|
<head>
|
|
<meta charset="utf-8" /><meta name="generator" content="Docutils 0.19: https://docutils.sourceforge.io/" />
|
|
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
|
<title>General Usage Examples — Fancy Gym 0.2 documentation</title>
|
|
<link rel="stylesheet" href="../_static/pygments.css" type="text/css" />
|
|
<link rel="stylesheet" href="../_static/css/theme.css" type="text/css" />
|
|
<link rel="stylesheet" href="../_static/style.css" type="text/css" />
|
|
<link rel="shortcut icon" href="../_static/icon.svg"/>
|
|
<!--[if lt IE 9]>
|
|
<script src="../_static/js/html5shiv.min.js"></script>
|
|
<![endif]-->
|
|
|
|
<script data-url_root="../" id="documentation_options" src="../_static/documentation_options.js"></script>
|
|
<script src="../_static/jquery.js"></script>
|
|
<script src="../_static/underscore.js"></script>
|
|
<script src="../_static/_sphinx_javascript_frameworks_compat.js"></script>
|
|
<script src="../_static/doctools.js"></script>
|
|
<script src="../_static/sphinx_highlight.js"></script>
|
|
<script src="../_static/js/theme.js"></script>
|
|
<link rel="index" title="Index" href="../genindex.html" />
|
|
<link rel="search" title="Search" href="../search.html" />
|
|
<link rel="next" title="DeepMind Control Examples" href="dmc.html" />
|
|
<link rel="prev" title="Gymnasium" href="../envs/open_ai.html" />
|
|
|
|
|
|
</head>
|
|
|
|
<body class="wy-body-for-nav">
|
|
<div class="wy-grid-for-nav">
|
|
<nav data-toggle="wy-nav-shift" class="wy-nav-side">
|
|
<div class="wy-side-scroll">
|
|
<div class="wy-side-nav-search" >
|
|
|
|
|
|
|
|
<a href="../index.html" class="icon icon-home">
|
|
Fancy Gym
|
|
<img src="../_static/icon.svg" class="logo" alt="Logo"/>
|
|
</a>
|
|
<div class="version">
|
|
0.2
|
|
</div>
|
|
<div role="search">
|
|
<form id="rtd-search-form" class="wy-form" action="../search.html" method="get">
|
|
<input type="text" name="q" placeholder="Search docs" aria-label="Search docs" />
|
|
<input type="hidden" name="check_keywords" value="yes" />
|
|
<input type="hidden" name="area" value="default" />
|
|
</form>
|
|
</div>
|
|
</div><div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="Navigation menu">
|
|
<p class="caption" role="heading"><span class="caption-text">User Guide</span></p>
|
|
<ul>
|
|
<li class="toctree-l1"><a class="reference internal" href="../guide/installation.html">Installation</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="../guide/episodic_rl.html">What is Episodic RL?</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="../guide/basic_usage.html">Basic Usage</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="../guide/upgrading_envs.html">Creating new MP Environments</a></li>
|
|
</ul>
|
|
<p class="caption" role="heading"><span class="caption-text">Environments</span></p>
|
|
<ul>
|
|
<li class="toctree-l1"><a class="reference internal" href="../envs/fancy/index.html">Fancy</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="../envs/dmc.html">DeepMind Control (DMC)</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="../envs/meta.html">Metaworld</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="../envs/open_ai.html">Gymnasium</a></li>
|
|
</ul>
|
|
<p class="caption" role="heading"><span class="caption-text">Examples</span></p>
|
|
<ul class="current">
|
|
<li class="toctree-l1 current"><a class="current reference internal" href="#">General Usage Examples</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="dmc.html">DeepMind Control Examples</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="metaworld.html">Metaworld Examples</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="open_ai.html">OpenAI Envs Examples</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="movement_primitives.html">Movement Primitives Examples</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="mp_params_tuning.html">MP Params Tuning Example</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="pd_control_gain_tuning.html">PD Control Gain Tuning Example</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="replanning_envs.html">Replanning Example</a></li>
|
|
</ul>
|
|
<p class="caption" role="heading"><span class="caption-text">API</span></p>
|
|
<ul>
|
|
<li class="toctree-l1"><a class="reference internal" href="../api.html">API</a></li>
|
|
</ul>
|
|
|
|
</div>
|
|
</div>
|
|
</nav>
|
|
|
|
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap"><nav class="wy-nav-top" aria-label="Mobile navigation menu" >
|
|
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
|
|
<a href="../index.html">Fancy Gym</a>
|
|
</nav>
|
|
|
|
<div class="wy-nav-content">
|
|
<div class="rst-content">
|
|
<div role="navigation" aria-label="Page navigation">
|
|
<ul class="wy-breadcrumbs">
|
|
<li><a href="../index.html" class="icon icon-home" aria-label="Home"></a></li>
|
|
<li class="breadcrumb-item active">General Usage Examples</li>
|
|
<li class="wy-breadcrumbs-aside">
|
|
<a href="https://github.com/ALRhub/fancy_gym/blob/release/docs/source/examples/general.rst" class="fa fa-github"> Edit on GitHub</a>
|
|
</li>
|
|
</ul>
|
|
<hr/>
|
|
</div>
|
|
<div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
|
|
<div itemprop="articleBody">
|
|
|
|
<section id="general-usage-examples">
|
|
<span id="example-general"></span><h1>General Usage Examples<a class="headerlink" href="#general-usage-examples" title="Permalink to this heading"></a></h1>
|
|
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="linenos"> 1</span><span class="kn">from</span> <span class="nn">collections</span> <span class="kn">import</span> <span class="n">defaultdict</span>
|
|
<span class="linenos"> 2</span>
|
|
<span class="linenos"> 3</span><span class="kn">import</span> <span class="nn">gymnasium</span> <span class="k">as</span> <span class="nn">gym</span>
|
|
<span class="linenos"> 4</span><span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
|
|
<span class="linenos"> 5</span>
|
|
<span class="linenos"> 6</span><span class="kn">import</span> <span class="nn">fancy_gym</span>
|
|
<span class="linenos"> 7</span>
|
|
<span class="linenos"> 8</span>
|
|
<span class="linenos"> 9</span><span class="k">def</span> <span class="nf">example_general</span><span class="p">(</span><span class="n">env_id</span><span class="o">=</span><span class="s2">"Pendulum-v1"</span><span class="p">,</span> <span class="n">seed</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">iterations</span><span class="o">=</span><span class="mi">1000</span><span class="p">,</span> <span class="n">render</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
|
|
<span class="linenos"> 10</span><span class="w"> </span><span class="sd">"""</span>
|
|
<span class="linenos"> 11</span><span class="sd"> Example for running any env in the step based setting.</span>
|
|
<span class="linenos"> 12</span><span class="sd"> This also includes DMC environments when leveraging our custom make_env function.</span>
|
|
<span class="linenos"> 13</span>
|
|
<span class="linenos"> 14</span><span class="sd"> Args:</span>
|
|
<span class="linenos"> 15</span><span class="sd"> env_id: OpenAI/Custom gym task id or either `domain_name-task_name` or `manipulation-environment_name` for DMC tasks</span>
|
|
<span class="linenos"> 16</span><span class="sd"> seed: seed for deterministic behaviour</span>
|
|
<span class="linenos"> 17</span><span class="sd"> iterations: Number of rollout steps to run</span>
|
|
<span class="linenos"> 18</span><span class="sd"> render: Render the episode</span>
|
|
<span class="linenos"> 19</span>
|
|
<span class="linenos"> 20</span><span class="sd"> Returns:</span>
|
|
<span class="linenos"> 21</span>
|
|
<span class="linenos"> 22</span><span class="sd"> """</span>
|
|
<span class="linenos"> 23</span>
|
|
<span class="linenos"> 24</span> <span class="n">env</span> <span class="o">=</span> <span class="n">gym</span><span class="o">.</span><span class="n">make</span><span class="p">(</span><span class="n">env_id</span><span class="p">)</span>
|
|
<span class="linenos"> 25</span> <span class="n">rewards</span> <span class="o">=</span> <span class="mi">0</span>
|
|
<span class="linenos"> 26</span> <span class="n">obs</span> <span class="o">=</span> <span class="n">env</span><span class="o">.</span><span class="n">reset</span><span class="p">(</span><span class="n">seed</span><span class="o">=</span><span class="n">seed</span><span class="p">)</span>
|
|
<span class="linenos"> 27</span> <span class="nb">print</span><span class="p">(</span><span class="s2">"Observation shape: "</span><span class="p">,</span> <span class="n">env</span><span class="o">.</span><span class="n">observation_space</span><span class="o">.</span><span class="n">shape</span><span class="p">)</span>
|
|
<span class="linenos"> 28</span> <span class="nb">print</span><span class="p">(</span><span class="s2">"Action shape: "</span><span class="p">,</span> <span class="n">env</span><span class="o">.</span><span class="n">action_space</span><span class="o">.</span><span class="n">shape</span><span class="p">)</span>
|
|
<span class="linenos"> 29</span>
|
|
<span class="linenos"> 30</span> <span class="c1"># number of environment steps</span>
|
|
<span class="linenos"> 31</span> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">iterations</span><span class="p">):</span>
|
|
<span class="linenos"> 32</span> <span class="n">obs</span><span class="p">,</span> <span class="n">reward</span><span class="p">,</span> <span class="n">terminated</span><span class="p">,</span> <span class="n">truncated</span><span class="p">,</span> <span class="n">info</span> <span class="o">=</span> <span class="n">env</span><span class="o">.</span><span class="n">step</span><span class="p">(</span><span class="n">env</span><span class="o">.</span><span class="n">action_space</span><span class="o">.</span><span class="n">sample</span><span class="p">())</span>
|
|
<span class="linenos"> 33</span> <span class="n">rewards</span> <span class="o">+=</span> <span class="n">reward</span>
|
|
<span class="linenos"> 34</span>
|
|
<span class="linenos"> 35</span> <span class="k">if</span> <span class="n">render</span><span class="p">:</span>
|
|
<span class="linenos"> 36</span> <span class="n">env</span><span class="o">.</span><span class="n">render</span><span class="p">()</span>
|
|
<span class="linenos"> 37</span>
|
|
<span class="linenos"> 38</span> <span class="k">if</span> <span class="n">terminated</span> <span class="ow">or</span> <span class="n">truncated</span><span class="p">:</span>
|
|
<span class="linenos"> 39</span> <span class="nb">print</span><span class="p">(</span><span class="n">rewards</span><span class="p">)</span>
|
|
<span class="linenos"> 40</span> <span class="n">rewards</span> <span class="o">=</span> <span class="mi">0</span>
|
|
<span class="linenos"> 41</span> <span class="n">obs</span> <span class="o">=</span> <span class="n">env</span><span class="o">.</span><span class="n">reset</span><span class="p">()</span>
|
|
<span class="linenos"> 42</span>
|
|
<span class="linenos"> 43</span>
|
|
<span class="linenos"> 44</span><span class="k">def</span> <span class="nf">example_async</span><span class="p">(</span><span class="n">env_id</span><span class="o">=</span><span class="s2">"fancy/HoleReacher-v0"</span><span class="p">,</span> <span class="n">n_cpu</span><span class="o">=</span><span class="mi">4</span><span class="p">,</span> <span class="n">seed</span><span class="o">=</span><span class="nb">int</span><span class="p">(</span><span class="s1">'533D'</span><span class="p">,</span> <span class="mi">16</span><span class="p">),</span> <span class="n">n_samples</span><span class="o">=</span><span class="mi">800</span><span class="p">):</span>
|
|
<span class="linenos"> 45</span><span class="w"> </span><span class="sd">"""</span>
|
|
<span class="linenos"> 46</span><span class="sd"> Example for running any env in a vectorized multiprocessing setting to generate more samples faster.</span>
|
|
<span class="linenos"> 47</span><span class="sd"> This also includes DMC and DMP environments when leveraging our custom make_env function.</span>
|
|
<span class="linenos"> 48</span><span class="sd"> Be aware, increasing the number of environments reduces the total length of the individual episodes.</span>
|
|
<span class="linenos"> 49</span>
|
|
<span class="linenos"> 50</span><span class="sd"> Args:</span>
|
|
<span class="linenos"> 51</span><span class="sd"> env_id: OpenAI/Custom gym task id or either `domain_name-task_name` or `manipulation-environment_name` for DMC tasks</span>
|
|
<span class="linenos"> 52</span><span class="sd"> seed: seed for deterministic behaviour</span>
|
|
<span class="linenos"> 53</span><span class="sd"> n_cpu: Number of cpus cores to use in parallel</span>
|
|
<span class="linenos"> 54</span><span class="sd"> n_samples: number of samples generated in total by all environments.</span>
|
|
<span class="linenos"> 55</span>
|
|
<span class="linenos"> 56</span><span class="sd"> Returns: Tuple of (obs, reward, done, info) with type np.ndarray</span>
|
|
<span class="linenos"> 57</span>
|
|
<span class="linenos"> 58</span><span class="sd"> """</span>
|
|
<span class="linenos"> 59</span> <span class="n">env</span> <span class="o">=</span> <span class="n">gym</span><span class="o">.</span><span class="n">vector</span><span class="o">.</span><span class="n">AsyncVectorEnv</span><span class="p">([</span><span class="n">fancy_gym</span><span class="o">.</span><span class="n">make_rank</span><span class="p">(</span><span class="n">env_id</span><span class="p">,</span> <span class="n">seed</span><span class="p">,</span> <span class="n">i</span><span class="p">)</span> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">n_cpu</span><span class="p">)])</span>
|
|
<span class="linenos"> 60</span> <span class="c1"># OR</span>
|
|
<span class="linenos"> 61</span> <span class="c1"># envs = gym.vector.AsyncVectorEnv([make_env(env_id, seed + i) for i in range(n_cpu)])</span>
|
|
<span class="linenos"> 62</span>
|
|
<span class="linenos"> 63</span> <span class="c1"># for plotting</span>
|
|
<span class="linenos"> 64</span> <span class="n">rewards</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">zeros</span><span class="p">(</span><span class="n">n_cpu</span><span class="p">)</span>
|
|
<span class="linenos"> 65</span> <span class="n">buffer</span> <span class="o">=</span> <span class="n">defaultdict</span><span class="p">(</span><span class="nb">list</span><span class="p">)</span>
|
|
<span class="linenos"> 66</span>
|
|
<span class="linenos"> 67</span> <span class="n">obs</span> <span class="o">=</span> <span class="n">env</span><span class="o">.</span><span class="n">reset</span><span class="p">()</span>
|
|
<span class="linenos"> 68</span>
|
|
<span class="linenos"> 69</span> <span class="c1"># this would generate more samples than requested if n_samples % num_envs != 0</span>
|
|
<span class="linenos"> 70</span> <span class="n">repeat</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">ceil</span><span class="p">(</span><span class="n">n_samples</span> <span class="o">/</span> <span class="n">env</span><span class="o">.</span><span class="n">num_envs</span><span class="p">))</span>
|
|
<span class="linenos"> 71</span> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">repeat</span><span class="p">):</span>
|
|
<span class="linenos"> 72</span> <span class="n">obs</span><span class="p">,</span> <span class="n">reward</span><span class="p">,</span> <span class="n">terminated</span><span class="p">,</span> <span class="n">truncated</span><span class="p">,</span> <span class="n">info</span> <span class="o">=</span> <span class="n">env</span><span class="o">.</span><span class="n">step</span><span class="p">(</span><span class="n">env</span><span class="o">.</span><span class="n">action_space</span><span class="o">.</span><span class="n">sample</span><span class="p">())</span>
|
|
<span class="linenos"> 73</span> <span class="n">buffer</span><span class="p">[</span><span class="s1">'obs'</span><span class="p">]</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">obs</span><span class="p">)</span>
|
|
<span class="linenos"> 74</span> <span class="n">buffer</span><span class="p">[</span><span class="s1">'reward'</span><span class="p">]</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">reward</span><span class="p">)</span>
|
|
<span class="linenos"> 75</span> <span class="n">buffer</span><span class="p">[</span><span class="s1">'terminated'</span><span class="p">]</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">terminated</span><span class="p">)</span>
|
|
<span class="linenos"> 76</span> <span class="n">buffer</span><span class="p">[</span><span class="s1">'truncated'</span><span class="p">]</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">truncated</span><span class="p">)</span>
|
|
<span class="linenos"> 77</span> <span class="n">buffer</span><span class="p">[</span><span class="s1">'info'</span><span class="p">]</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">info</span><span class="p">)</span>
|
|
<span class="linenos"> 78</span> <span class="n">rewards</span> <span class="o">+=</span> <span class="n">reward</span>
|
|
<span class="linenos"> 79</span>
|
|
<span class="linenos"> 80</span> <span class="n">done</span> <span class="o">=</span> <span class="n">terminated</span> <span class="ow">or</span> <span class="n">truncated</span>
|
|
<span class="linenos"> 81</span> <span class="k">if</span> <span class="n">np</span><span class="o">.</span><span class="n">any</span><span class="p">(</span><span class="n">done</span><span class="p">):</span>
|
|
<span class="linenos"> 82</span> <span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Reward at iteration </span><span class="si">{</span><span class="n">i</span><span class="si">}</span><span class="s2">: </span><span class="si">{</span><span class="n">rewards</span><span class="p">[</span><span class="n">done</span><span class="p">]</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
|
<span class="linenos"> 83</span> <span class="n">rewards</span><span class="p">[</span><span class="n">done</span><span class="p">]</span> <span class="o">=</span> <span class="mi">0</span>
|
|
<span class="linenos"> 84</span>
|
|
<span class="linenos"> 85</span> <span class="c1"># do not return values above threshold</span>
|
|
<span class="linenos"> 86</span> <span class="k">return</span> <span class="o">*</span><span class="nb">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">v</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">stack</span><span class="p">(</span><span class="n">v</span><span class="p">)[:</span><span class="n">n_samples</span><span class="p">],</span> <span class="n">buffer</span><span class="o">.</span><span class="n">values</span><span class="p">()),</span>
|
|
<span class="linenos"> 87</span>
|
|
<span class="linenos"> 88</span>
|
|
<span class="linenos"> 89</span><span class="k">if</span> <span class="vm">__name__</span> <span class="o">==</span> <span class="s1">'__main__'</span><span class="p">:</span>
|
|
<span class="linenos"> 90</span> <span class="n">render</span> <span class="o">=</span> <span class="kc">True</span>
|
|
<span class="linenos"> 91</span>
|
|
<span class="linenos"> 92</span> <span class="c1"># Basic gym task</span>
|
|
<span class="linenos"> 93</span> <span class="n">example_general</span><span class="p">(</span><span class="s2">"Pendulum-v1"</span><span class="p">,</span> <span class="n">seed</span><span class="o">=</span><span class="mi">10</span><span class="p">,</span> <span class="n">iterations</span><span class="o">=</span><span class="mi">200</span><span class="p">,</span> <span class="n">render</span><span class="o">=</span><span class="n">render</span><span class="p">)</span>
|
|
<span class="linenos"> 94</span>
|
|
<span class="linenos"> 95</span> <span class="c1"># Mujoco task from framework</span>
|
|
<span class="linenos"> 96</span> <span class="n">example_general</span><span class="p">(</span><span class="s2">"fancy/Reacher5d-v0"</span><span class="p">,</span> <span class="n">seed</span><span class="o">=</span><span class="mi">10</span><span class="p">,</span> <span class="n">iterations</span><span class="o">=</span><span class="mi">200</span><span class="p">,</span> <span class="n">render</span><span class="o">=</span><span class="n">render</span><span class="p">)</span>
|
|
<span class="linenos"> 97</span>
|
|
<span class="linenos"> 98</span> <span class="c1"># # OpenAI Mujoco task</span>
|
|
<span class="linenos"> 99</span> <span class="n">example_general</span><span class="p">(</span><span class="s2">"HalfCheetah-v2"</span><span class="p">,</span> <span class="n">seed</span><span class="o">=</span><span class="mi">10</span><span class="p">,</span> <span class="n">render</span><span class="o">=</span><span class="n">render</span><span class="p">)</span>
|
|
<span class="linenos">100</span>
|
|
<span class="linenos">101</span> <span class="c1"># Vectorized multiprocessing environments</span>
|
|
<span class="linenos">102</span> <span class="c1"># example_async(env_id="HoleReacher-v0", n_cpu=2, seed=int('533D', 16), n_samples=2 * 200)</span>
|
|
</pre></div>
|
|
</div>
|
|
</section>
|
|
|
|
|
|
</div>
|
|
</div>
|
|
<footer><div class="rst-footer-buttons" role="navigation" aria-label="Footer">
|
|
<a href="../envs/open_ai.html" class="btn btn-neutral float-left" title="Gymnasium" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left" aria-hidden="true"></span> Previous</a>
|
|
<a href="dmc.html" class="btn btn-neutral float-right" title="DeepMind Control Examples" accesskey="n" rel="next">Next <span class="fa fa-arrow-circle-right" aria-hidden="true"></span></a>
|
|
</div>
|
|
|
|
<hr/>
|
|
|
|
<div role="contentinfo">
|
|
<p>© Copyright 2020-2024, Fabian Otto, Onur Celik, Dominik Roth, Hongyi Zhou.</p>
|
|
</div>
|
|
|
|
Built with <a href="https://www.sphinx-doc.org/">Sphinx</a> using a
|
|
<a href="https://github.com/readthedocs/sphinx_rtd_theme">theme</a>
|
|
provided by <a href="https://readthedocs.org">Read the Docs</a>.
|
|
|
|
|
|
</footer>
|
|
</div>
|
|
</div>
|
|
</section>
|
|
</div>
|
|
<script>
|
|
jQuery(function () {
|
|
SphinxRtdTheme.Navigation.enable(true);
|
|
});
|
|
</script>
|
|
|
|
</body>
|
|
</html> |