Stanford Online/ DeepLearning.AI. Unsupervised Learning, Recommenders Systems and Reinforcement Learning: State - Action Value Function Example

State-action value function example.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# State Action Value Function Example\n",
    "\n",
    "In this Jupyter notebook, you can modify the mars rover example to see how the values of Q(s,a) will change depending on the rewards and discount factor changing."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np   # Get numpy 'np' constructor for arrays handling and numeric computations\n",
    "from utils import *  # Import ALL '*' helper functions from utils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Do not modify\n",
    "num_states = 6    # states s = 1,2,3,4,5,6\n",
    "num_actions = 2   # action a = <- go left. action a = -> go right "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "terminal_left_reward = 100  # reward = 100 at terminal state s = 1 \n",
    "terminal_right_reward = 40  # reward = 40 at terminal state s = 6\n",
    "each_step_reward = 0        # reward = 0 at others states s = 2,3,4,5\n",
    "\n",
    "# Discount factor\n",
    "gamma = 0.5                 # gamma = 0.5\n",
    "\n",
    "# Probability of going in the wrong direction\n",
    "misstep_prob = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAqsAAACNCAYAAACQcTPzAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAcrklEQVR4nO3deXhU5d3/8fcNSSAhISQQIwEqIAEFFHEBKWBYLGqt6IP2qVaeVqvVR6tVcKl9oOjPYtUH15+/i6VKrbsFbUsE3NgRxIVFUASJbAkIEpYEErLfvz/OZEwyk2RCQuaczOd1XXMlc597zrlPPtecfOfMPWeMtRYRERERETdqFe4BiIiIiIjURsWqiIiIiLiWilURERERcS0VqyIiIiLiWipWRURERMS1VKyKiIiIiGupWBWRsDDGjDHGvGuMOWiMKTLGfGOMedwYk9SIdd5tjBkXpP0hY0yzX6cvjNvdaYz5e5X7NxhjrDGme3OPRUSksVSsikizM8b8D/A+UATcDFwCzARuAD4zxnQ7wVXfDQQUq8ALwJATXGdLsABn/78L90BERBoqKtwDEJHIYowZCUwFnrHWTqiyaLkx5l/AWuBlYGRTbdNamwPkNNX6vMZaewA4EO5xiIicCJ1ZFZHmdj9wCPhjzQXW2h3AY8AIY8zgynbfW9iPGGMmGWNyjDHHjTErjDHnVOmzEzgNuN7X31a+FR7s7Xjf8qnGmHuMMbuMMQXGmAXGmFN8tznGmDxjTLYx5g81HptijJnlm7pQ6OvzujGmy4n8QXxv279qjPmtMSbLNy1ina+wr9l3vDHmC1+fXGPMK8aYzvWsP+g0AN/21vn+noeNMcuNMT82xrQxxhwwxjxdx7rOOJF9FRFpKBWrItJsjDFRQAbwobW2qJZumb6fo2q0/wr4KXAHznSBVGCxMSbZt/w/gH040wuG+G5/rmdI/+Xbzu3AncBwnLO6/wI2AlcDC4HHjDE/rfK4ZJwpDH8ELgXuA9KBVcaYtvVsszYZwERgEnAtUAy8a4zpU9nBGHML8ArwNc50hwdwplAsN8bEN2RjxpgngL8C64D/BMYDK4AfWWuLgReBXwfZn1uB5dbaLQ3eQxGRE6BpACLSnDoCscDOOvpULqs5bzUWGGOtLQAwxnwCbAMmAH+y1q43xhQDudbaNSGOpxi40lpb5ltn/yrrm+prW4ZTCP8cp3DFWrsVuKtyJcaY1sAqYDdwGU6x21CpwFBr7W7fOhcDu4DJwH/5tvFnYJm19toq294CrAR+A/zfUDZkjOnl28+nrbUTqyxaUOX3GcA9OPv9iu9xZwMXAtedwP6JiJwQnVkVkeZkGvHYhZWFKoC1diewhsZ9cOrDykLVp/Js4ftVtlMGZFGjeDbG3OZ7O/4YUIZTqAL04cSsqSxUfds9yg8fjKpc7ynAa1UfZK39CKeozWjAti7GOf7/tbYOvikZ7+OcSa10K87c1382YFsiIo2iYlVEmlMucBzoXkefymXZNdr3B+m7HziheaI+h2vcL6mj3f92uDHmTmA6sAjn7fhBOGccqdqvgerbv8rpDsE+0b+vyvJQdPT9rO9DZ9OBocaY/saYdjhTBV601pbU8zgRkSajaQAi0mystWXGmBXAT4wxbWuZtzrW93NJjfbUIH1TgT1NOcYQXQssttbeU9lgjOnRyHXWt3+HfD9PDdLvVODzBmwr1/ezC7C1jn4LcaZl3Ap8ASRQx9lYEZGTQWdWRaS5TcM5s/eXmgt8Bd8fgBXW2k9qLP6p7+xeZd/uOGczP67SpxhnbuvJFgeU1mi7sZHrvLDq9WWNMQnA5fywf1txzrReW/VBxpgf41wFYXkDtrUIqABuqauTtbYCmIXzQbQ7gEXW2m8bsB0RkUbTmVURaVbW2sXGmCnAw76C82Wct93Pxfl0ex5OcVTTceADY8w0oA3wf4B8oOrllTYDw40xP8N5azzXN7e1qb0H/MH35Qaf4lxR4JpGrnM/zv49hFN0/wFoh++KBtbact/fbZYx5lXgVZwzo4/gfNDsxVA3ZK391ndZqom+ojgTKMeZzrDFWvuPKt1nAw8BA3CujiAi0qxUrIpIs7PW/tkY8xnOJ9JfxDlTuRuncH3UWnsoyMNeBgqA/wd0Aj4Drq3R94/A88AcnDOsL+Fc5qqpPQx08I2/Lc5ZzUuA7Y1Y53JgGc4Z5644hfdl1tpvKjtYa/9qjCnEuVTWPOAYzlv191trjzVkY9bae40xWTiX7fo1zt92I/BBjX4HjDHLgbP44bJiIiLNxljb7F9bLSLSIL4L+j9irZ0c7rGcDL4vNPjIWjs+3GOpyRiThPNC4hlr7Z/CPR4RiTw6syoiIgGMMSk4l8u6C+fzDdPDOyIRiVT6gJWIiARzOc6XDQwCfm2tDXbJLBGRk07TAERERETEtXRmVURERERcS8WqiIiIiLhWnR+wio6OtgMGDGiusUgT2rrV+VKaPn1O9GvKJVyUnbcpP29Tft6l7Lxv7dq1udbalJrtdc5ZTUhIsEePHj2pA5OTY8SIEQAsW7YsrOOQhlN23qb8vE35eZey8z5jzFpr7fk12zUNQERERERcS8WqiIiIiLiWilURERERcS0VqyIiIiLiWipWRURERMS1VKyKiIiIiGupWBURERER11KxKiIiIiKupWJVRERERFxLxaqIiIiIuJaKVRERERFxLRWrIiIiIuJaKlZFRERExLVUrIqIiIiIa6lYFRERERHXUrEqIiIiIq6lYlVEREREXEvFqoiIiIi4VlS4BxCq4999R9asWeR9+SV5X39NRVERo5cvJ65r12r9SvLy+Pqxx/juww+pKCoiaeBA+k2eTPs+far1Ky8uZutTT5Ezbx6l+fkk9u3LmfffT8dBg0Iaz6433+Tb2bM5npNDbJcu9PzNb+j+y1822f62dLlr1vDx9dcHtEclJHDZhg3++6HmGYytqCBr1ix2vfEGxQcOEN+zJ+l33knapZc26b5Eor3vvsued94hb9Mmig8eJDYtjc6XXEL6bbcRFR8PQGFODoszMoI+/tL164lu377ObSi/kyeU4+mBVavIfvttDq9fT9H+/bRNTSVl2DD63HUXbTp1qncbiy66iON79gS0nz9jBp3HjGnS/YkkoTz3QMdOr1hzww0cWLmS9Ntv54x77vG3K7/qPFOsFuzaxd6FC+nQvz8dL7iAAytXBvSx1vLZLbdQmJPDWVOmEJ2YyLaZM1l9/fVkvPMOsZ07+/t+8cAD7F+6lL4PPEBct27sfPVV1txwA8PeeovEvn3rHMuuN99k4+TJ9Prv/yZl6FByV69m05QpYC3dgxRgUrv+U6bQ4eyz/fdN69b+3xuSZzBbnnqK7bNnc8bEiST278+e+fNZe8cdtH7+eVJHjjxp+xQJvn3hBWLT0jjj3ntpe+qp5G/ezNZnnyV3zRqGzZ2LafXDmza9bruNU0ePrvb4qHbt6t2G8jt5Qjme7nrjDcoKCkj/3e+I69aNgp07+ebZZzmwciUZCxaElGHK8OH0ueuuam3tevZssv2IRKE893Ts9IY9mZnkb9kS0K78grDW1nqLj4+3blFRXu7/feebb9rMnj1tQXZ2tT7fffCBzezZ0x5YvdrfVpKfb98dONBueughf9uRzZttZs+edtfcuf628tJSu3j0aPvJb39b5zjKS0vte+efb9fdc0+19vX332/fPe88W15SckL719QyMjJsRkZGuIdRqwMff2wze/a033/0Ua19Qs0zmKIDB+z8M86wW55+ulr76uuvt0svu6xxgz/J3J6dtdYW5eYGtO1++20nr1WrrLXWFmRn28yePe3ON99s+PqV30kVyvE0WMa5n3ziHDvnzKl3Gx8OH27XTpjQ+ME2M7fnF8pzT8dO9yvJy7PvDxpkc+bNs5k9e9qvn3jCvyxS87PWWuBzG6Qe9cyc1apnamqzb/Fi2qam0mnIEH9bdEICqaNGsW/RIn/b/kWLMNHRpF1+ub+tVVQUXX72Mw6sXEl5cXGt2zi8fj0lhw7R9aqrqrV3veoqSg8f5tDnnzdkt6QOoeYZzPcrV1JRUkKXK6+s1t7lqqs4unUrhdnZJ2XMkaJNx44BbZVnyI/v39/o9Su/kyuU42ldGRft29fkY5LQhPLc07HT/TY/9hgJ6el0GTs2YJnyC+SZYjUUR7dtI6F374D2hPR0ju/dS1lBgb9fXNeuRMXGBvSrKCmhcNeuOrcBBGwnIT3dWZ6V1ah9iDTrJkzgnfR03jvvPNbdfTeFe/f6l4WaZzDHtm2jVUwM7bp3D3gsKKeT4eCnnwKQcPrp1dq3PPEE83v35t0BA/j0llvI37q13nUpP3c6+MknAMT36hVS//2LF7OgXz8WnHkmK6++mu8++OBkDi9i1Xzu6djpbgc//5ycf/2Lsx5+OOhy5RfIM3NWQ1F65AhxXboEtEd36OAsz8sjql07SvLyiE5MrLVfSV5endsAAh7v34ZvudQtOiGBnjfdRMfBg4mKjyd/82a2TZ/OwWuu4aLMTNp06hRynsGUHDlCdPv2GGOqtcf4citRTk3q+L59bH36aToNHeo/y9MqJobTrruOlOHDiUlO5ti337Jtxgw++vnPGf7Pf5JQR8Gj/Nyn7Ngxvpw6lfhevTj1Jz+pt3/qqFF0OPts4rp1ozg3l52vvMLnt93GwCefDHhnSk5csOeejp3uVVFaysbJkzn95puJr2X+tvIL1KKKVayFGgH522vcrxlk0H5BN1F/H6lfYr9+JPbr57/fafBgki+4gI/GjWPHSy85n4oMNc9ganms8mt6ZQUFfHbrrZioKM55/HF/e9tTTuHsqVP99ztecAGnXHQRyy67jG3Tp3PuU0/VvlLl5yoVZWWsvftuivbvZ9icObSKqv9fx1kPPVTtfucxY1h59dV8PW2aitUmUttzT8dO98qaNYuKoiLSf/e72jspvwAtahpAdIcOlAY5K1rZVnk2NCYxMeiri8p+MUHOulaKqfLKptpjK8+4+pZLw3Xo3592PXpwZNMmIPQ8g6l8bM0naGl+PvBDjtI45cXFfHrLLRRmZ3Ph3/9e76dUY9PSSD7vPI5s3FhnP+XnHraigg333UfuqlVcMGMG7c8444TWY1q3Ju2yyyjat4+i779v4lFGnrqeezp2ulPh3r1smz6dPhMmUFFSQml+vv/vWnnflpcrvyBaVLGakJ7un1Na1dGsLGLT0vynzRN696YwJ4ey48cD+rWKiSHutNPq3AYQsJ3KeSB1vbUpIajyBAs1z2Bqm398rHLOsXJqtIrSUj6//XaObNzI4NmzQ7r+H/he4Qc7a1CF8nOPjZMns3fBAs599llShg5tmpXWk7/Urb7nno6d7lS4ezcVxcWsnziR9wYO9N/AuSTZewMHkr91q/ILokUVq6mjR1O0bx+5vg8BAJQePcr+JUtIrXKdx9TRo7GlpXy3cKG/raKsjL0LFpAybBit27SpdRtJAwcSk5zMnnnzqrXvmTeP6A4dSD7vvCbco8hyZONGju3YQdI55wCh5xnMKRddRKuYGHIyM6u158ybR0Lv3sR169b0OxBBbEUF6yZOJHf1ai6YOZMk3wG3PoV793J43TqSBgyos5/yc4ev/vIXds+Zw4DHH2/0hfwrysrYu3AhsWlptE1JaaIRRp5Qnns6drpTYt++DHnttYAbOJ/WH/Laa7Q77TTlF4Sn5qzuffddAPK+/BKA75cvJyY5mZjkZDoNHsypF19M0sCBrJ84kb4PPEB0YiJZM2eCtfS65Rb/ehL79iXt8sv5aupUKsrKiOvalV2vv05hdjYDa8yjWzxyJHFdujDk1VcBaBUdTZ8JE9g0ZYpzaYmhQ8n9+GN2z51L/wcfpFVMTDP9Nbxt3YQJxHXtSmL//kQnJJC3eTPbZs6kbWoq3X/1K4CQ8wSY37s3XceN45zHHgOgTadO9LzxRrJmzCCqXTsS+/Vj74IF5H78MRfMmtXs+9vSbHrwQb5buJD0228nKi6Ow+vX+5e1PfVUYjt35qu//AVbUUGy7wXesR07yJoxA4wh/fbbq61P+TW/+o6nWbNmsX32bLr9/OfEd+9eLeOY5GTaVXkHqmZ+ezIz2bdoEaeMGEFs587OB6xefZW8L7/k3Geeaca9bHlCee7p2OlO0e3b0+nCC4Mui0tL8y9TfoE8VayuveOOavc3TZkCQMfBg+n0+uuYVq0Y9MILbH70UTY9+CDlxcUkDxzIkNdeIzYtrdpjz/nf/2XLk0+y9amnKM3Pp/2ZZzL4xRfp0L9/tX62vBxbUVGtrfJrVbfPnu18m0jnzpz10EN0Hz++qXe5xUro3Zs977zDjpdfpryoiDadOtF5zBj63H03bZKTARqUpy0vx5aXV2s74557aB0Xx46//53i3Fza9ejBec89F/BtStJw3y9fDsC26dPZNn16tWW9f/97+tx1Fwnp6ex87TVy3n6bsoICYpKS6DRkCL1///uAT8Eqv+ZX3/G0MuPsuXPJnju3Wt+u48YxcNo0//2a+cV160bxwYNsfuwxSvPyaN22LR3OPpvBL77IKRdddLJ2KSKE8tzTsdPblF8gU9cnxBISEuzRo0ebcTjSVEaMGAHAsmXLwjoOaThl523Kz9uUn3cpO+8zxqy11p5fs71FzVkVERERkZZFxaqIiIiIuJaKVRERERFxLRWrIiIiIuJaKlZFRERExLVUrIqIiIiIa6lYFRERERHXUrEqIiIiIq6lYlVEREREXEvFqoiIiIi4lopVEREREXEtFasiIiIi4loqVkVERETEtVSsioiIiIhrqVgVbzm2M9wjkMZQft6l7CJHeTEc3xfuUYj4qVgV7zi0Ft4dCJ//PtwjkROh/LxL2UWO8mJYcSV8OBwK94Z7NCKAilXxikPrYMlPoPQIfPMc7PpHuEckDaH8vEvZRZYv/wzfvQ/HsmDxSBWs4goqVsX9Dm9w/lmWHHbun/5b+NF/hndMEjrl513KLvL0+x9IHen8fvQbWDIKjn8X3jFJxFOxKu52+AtYcjGUHHLun/5bGDQLjAnvuCQ0ys+7lF1kioqDjPk/FKz5W2HxKM1hlbBSsSrudXgjLBkNxQed+/pn6S3Kz7uUXWQLKFi3+ArW/eEdl0SsqHAPQCSovK+r/7ME+H45LDizadb/sy1Nsx4JTvl5l7LztvxtsOKKpllXeVGV9X7tTAm4eAW06dg06xcJkYpVcad9i6E4t3rb0W/CMxZpOOXnXcrO2yqKnbfuT4a8zXBkE6SOODnrF6mFpgGIOyWfC63ahHsUcqKUn3cpO6lN21Mg/vRwj0IikM6sijul/Bgu+hes+A/nTAHAwCfgzHvCOy4JjfLzLmXnbR36wy9t49djLXxyE2x/0bnfJgVGLYF23Rq/bpEG0plVca+0y2D429Aqxrm//l74+snwjklCp/y8S9lFtoBCtROMXgwd+oV3XBKxVKyKu3W5XP80vUz5eZeyi0wBhWpHGLUYOpwV3nFJRFOxKu7X5WcwbC60inbur78Xsv8Z3jFJ6JSfdym7yPPlw4GFatLZ4R2TRDwVq+INXcfC0DnOP820y52beIfy8y5lF1nSb4fE/hCTDKMWQdKAcI9IRB+wEg/pdpXzKr/jIGitTyt7jvLzLmUXOdqmwOglzles6oyquISKVfGWU4aHewTSGMrPu5Rd5Gib4txEXELTAERERETEtVSsioiIiIhrqVgVEREREddSsSoiIiIirqViVURERERcS8WqiIiIiLiWilURERERcS0VqyIiIiLiWipWRURERMS1VKyKiIiIiGupWBURERER12oxxWpOTg533nknQ4YMIS4uDmMMO3fuDOh3+PBhbr75Zjp16kS7du24+OKL2bRpU0C/oqIi7rvvPjp37kxsbCxDhgxhxYoVzbAnUpfs7GyuueYaEhMTad++PePGjWP37t3hHpaEQNl5m/LzLmXnXZdeeinGGCZPnlytPdRapqVoMcVqVlYWc+bMISkpieHDhwftY61l7NixvPfeezz33HO8/fbblJaWMnLkSHJycqr1vemmm3j++ed5+OGHmT9/Pp07d+aSSy5hw4YNzbE7EkRhYSGjRo1iy5YtvPTSS7zyyits27aNkSNHUlBQEO7hSR2UnbcpP+9Sdt71xhtv8MUXXwS0N6SWaTGstbXe4uPjrVeUl5f7f3/++ectYHfs2FGtz7///W8L2CVLlvjbjhw5YpOSkuydd97pb9uwYYMF7N/+9jd/W2lpqe3du7e94oorTt5ONKGMjAybkZER7mE0qWeeeca2atXKbtu2zd+2fft227p1a/vkk0+GcWRNS9l5m/LztpaWn7LzpsOHD9vU1FT7+uuvW8BOmjTJvyzUWsaLgM9tkHq0xZxZbdWq/l3JzMwkLS2NkSNH+tsSExO54oormDdvXrV+0dHR/OIXv/C3RUVFce211/L+++9TXFzctIOXkGRmZnLhhRfSq1cvf1uPHj0YOnRotfzEfZSdtyk/71J23nT//ffTr18/rrvuuoBlodYyLUmLKVZD8dVXX9G/f/+A9n79+rF7926OHTvm79ejRw/i4uIC+pWUlJCVldUs45Xq6spv8+bNYRiRhErZeZvy8y5l5z0fffQRL7/8MtOnTw+6PNRapiWJqGL10KFDJCUlBbQnJycDzoTlUPodOnToJI5SalNXLpXZiTspO29Tft6l7LyltLSUW2+9lXvvvZc+ffoE7RNqLdOSRFSxaq3FGBO0/UT6SfNTLt6l7LxN+XmXsvOOxx9/nOPHjzNp0qRa+0RijRIV7gE0p+Tk5KBnRStfhVS+UklOTg56WY/KfpWvXqR5JSUl1ZpfsFeZ4h7KztuUn3cpO+/YvXs3jzzyCC+88ALFxcXVPh9TXFzMkSNHSEhICLmWaUki6sxqv379+OqrrwLaN2/ezI9+9CPi4+P9/Xbs2EFhYWFAv5iYmGoT1aX51JVf3759wzAiCZWy8zbl513Kzju2b99OUVER48ePJykpyX8DeOKJJ0hKSmLTpk0h1zItSUQVq2PHjmXPnj0sX77c35afn88777zD2LFjq/UrLS1l7ty5/raysjL+8Y9/MGbMGNq0adOs4xbH2LFjWbNmDdu3b/e37dy5k1WrVlXLT9xH2Xmb8vMuZecd55xzDkuXLg24AYwfP56lS5fSq1evkGuZlsTUNcchISHBHj16tBmH0zhvvfUWAIsXL2bmzJlMnz6dlJQUUlJSyMjIoKKigmHDhpGdnc20adNISkri0UcfZePGjXzxxRd069bNv67Ky1RNmzaNHj16MGPGDObPn8/q1as599xzw7WLIRsxYgQAy5YtC+s4mlJBQQEDBgwgNjaWqVOnYozhT3/6E0ePHmXjxo0t5tWksvM25edtLS0/Zed9xhgmTZrE1KlTARpUy3iNMWattfb8gAXBLr5qPfilANZaCwS9Vb1I8MGDB+2NN95ok5KSbGxsrB01apTdsGFDwLoKCwvthAkTbGpqqm3Tpo0dNGiQXbp0afPtTCO1pIsjV7Vr1y47btw4m5CQYOPj4+2VV14Z8OUPXqfsvE35eVtLzE/ZeRs1vhTA2tBrGa+hli8FaFFnVuUHLfUVZiRQdt6m/LxN+XmXsvO+2s6sRtScVRERERHxFhWrIiIiIuJaKlZFRERExLVUrIqIiIiIa6lYFRERERHXUrEqIiIiIq6lYlVEREREXEvFqoiIiIi4lopVEREREXEtFasiIiIi4loqVkVERETEtVSsioiIiIhrqVgVEREREddSsSoiIiIirqViVURERERcS8WqiIiIiLiWilURERERcS0VqyIiIiLiWipWRURERMS1VKyKiIiIiGupWBURERER11KxKiIiIiKuZay1tS805gCwq/mGIyIiIiIR6jRrbUrNxjqLVRERERGRcNI0ABERERFxLRWrIiIiIuJaKlZFRERExLVUrIqIiIiIa6lYFRERERHX+v97cawaR0piYAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 864x144 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA/oAAACNCAYAAAAHKQFFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAeNUlEQVR4nO3deXwV5b3H8e+D2RcCIWwBJAQTEClKBZE1IC1KvbVW6dV7qYq1WlFRFtt676WCFilWxe2W5QVebHGhoreyCZcKKLsIUhHBALJI2NcEAklI8tw/kpzmJCfJOZBwJnM+79frvMh55pmZ55nnNzP8zpmZY6y1AgAAAAAA7tAg2A0AAAAAAAC1h0QfAAAAAAAXIdEHAAAAAMBFSPQBAAAAAHAREn0AAAAAAFyERB8AAAAAABch0QcAAAAAwEVI9AEAcChjzCBjzGJjzAljTJ4xJtMYM8kY08hH3VbGmFxjTLfL1LZRxpgtxhj+LwEAgMNwcgYAwIGMMf8p6f8k5Un6paSbJU2XdL+kDcaYVhVm+b2kFdbajZepidMkNZN032VaHwAA8JOx1ga7DQAAoBxjzABJyyS9aq0dVWFaO0mbJG201g4qLWsuab+kn1prF13Gdv5R0q3W2msu1zoBAEDN+EYfAADn+Y2kk5L+o+IEa+0eSZMk/dAYc31p8TBJZ1RyBYCHMeZmY8xaY0y2MeZs6aX/T1e3YmNMU2PMdGPMDmPMOWPMfmPMOz6uIJCkOZI6GWN6Bd5FAABQV0j0AQBwEGNMmKQMSX+31uZVUW1+6b8DS/+9RdI6a21hueWkltbbI+kuSbdJmiwptoYmJKrkdoH/KF3uryWlSVpjjImqUPcfknJK6wEAAIcIC3YDAACAlyaSoiXtraZO2bS2xhgjqYeklyvU+b6kCEnDrbU5pWXLa1q5tTZT0hNl740xV0haI+k7SYMl/a1c3WJjzBZJN9a0XAAAcPnwjT4AAM5iAqhbLKmRSj4YOFZh2j8kXZA0xxgzxBjTzO8GGDPcGPOlMeaspEKVJPmS1MFH9WOSkgNoMwAAqGMk+gAAOMtxSeclpVRTp2zaAUlll9Pnl69grd2lkif1N5A0W9JhY8xnxpiM6lZujBkhaYqkjyXdIekG/fMb+4qX7qu0rdHVLRMAAFxeJPoAADhI6X32K1XysD1fibVUcr+9JH0q6UTp3419LGuFtfYWlXzr/wOVfMO/yBiTVE0T7pa0zFo7xlq71Fr7uaSj1dRPVMmHEwAAwCFI9AEAcJ4XVHKv/sSKE0p/Xu+3kr601q6z1hao5IF7qVUtzFqbb61dLumPKnkYX7tq1h2jkg8Eyru/mvrtJGVWMx0AAFxmPIwPAACHsdYuK/0ZvGeNMSmS/iLplEoesPeUSj6ov6vcLCtVcom9hzHmYUn9JH0kab+kJJU8Sf+gpK2ldVJU8iHBM9ba8aWzLpH0W2PMf0raIOkmSUN8tdMY00hSuqQXL763AACgtpHoAwDgQNba3xtjPpc0StIslVx+L0kbJf3UWptVrvpfJd1rjEmx1u4tLftSJU/J/4OkZpJOSlotaai19nxpnbKf2jtcblnPlq5rlEruyf9UJff67/bRzFslFajck/gBAEDwGWttsNsAAAD8YIx5S9JPJQ201q4vV95A0k5Js6y1EwJY3kOSnpPU1lp77iLas1jScWvtPYHOCwAA6g6JPgAA9YQxJkLSUknfk9THWru93LShkiZLaudv0m6MeVvS19baSs8C8GPe6yStl9S59An/AADAIUj0AQBwAWOMkfRrSQuttdsuw/pukdTYWvtuXa8LAAAEhkQfAAAAAAAX4ef1AAAAAABwkWqfuh8eHm6vvfbay9UWwG+ZmSU/2dyhQ4cgtwTwRmzCqYhNOBnxCaciNuF0mzZtOm6tbVqxvNpL9+Pj4+2ZM2fqtGHAxejfv78k6ZNPPglqO4CKiE04FbEJJyM+4VTEJpzOGLPJWtutYjmX7gMAAAAA4CIk+gAAAAAAuAiJPgAAAAAALkKiDwAAAACAi5DoAwAAAADgIiT6AAAAAAC4CIk+AAAAAAAuQqIPAAAAAICLkOgDAAAAAOAiJPoAAAAAALgIiT4AAAAAAC5Cog8AAAAAgIuQ6AMAAAAA4CIk+gAAAAAAuAiJPgAAAAAALkKiDwAAAACAi5DoAwAAAADgIiT6AAAAAAC4CIk+AAAAAAAuQqIPAAAAAICLkOgDAAAAAOAiJPoAAAAAALgIiT4AAAAAAC5Cog8AAAAAgIuQ6AMAAAAA4CIk+gAAAAAAuAiJPgAAAAAALkKiDwAAAACAi5DoAwAAAADgIiT6AAAAAAC4CIk+AAAAAAAuQqIPAAAAAICLkOgDAAAAAOAiJPoAAAAAALhIWG0v8PyhQ9o1fbqyt25V9vbtKs7L08BPP1VM69Ze9Qqys7V90iQd+vvfVZyXp8Zdu+qasWPVsEMHr3pF+fnKnDxZWfPm6UJOjhI6ddLVv/mNmtxwg1/t2Tdnjr594w2dz8pSdKtWSv3FL5Ty7/9ea/11OsajesfXr9e6oUMrlYfFx2vwP/7hee/v9vHFFhdr1/Tp2vfuu8o/dkxxqalKGzFCybfcUqt9cQt/YvbYmjXa/8EHOrV5s/KOHFFU8+Zq2qePOjzxhCKTkmpcx8f9+un8gQOVyrtNnaqWgwbVan8u1sHFi3VgwQJlf/WV8k+cUHRyslrefLPShg9XWFycJOlcVpaWZWT4nP+WzZsV3rBhtesgNgN3ZMWKkvj8+muZBg0Um5KiTr/9rZJ69fJZ//SWLdo3Z45OfP65zh88qIjGjdWke3d1HD1aMW3aeNWtD3FZhv3UWWr7XO9LfTtesK86gz/nMim0YrO89cOG6diqVUp75BF1HDPGUx6q2yPY3DYetZ7o5+7bp4MffaRGnTurSffuOrZqVaU61lp9/tBDOpeVpe89/bTCExK0c9o0rR06VBkLFii6ZUtP3S+fekpHVqxQp6eeUkybNtr71ltaP2yY+rz/vhI6daq2LfvmzNGWsWN11cMPq2nv3jq+dq2+evppyVql+Eju3Ijx8E/np59Woy5dPO/NFVd4/g5k+/jyzeTJ2v3GG+o4erQSOnfWgYULtemxx3TFjBlqPmBAnfWpvvInZve9+64Kc3OV9uijimnTRrl792rHq6/q2KpVyli0SGGxsTWup2nfvurwxBNeZbGpqbXWj0v17cyZik5OVscnn1RUixbK2bZNma++quPr16vP3LkyDf55QdZVw4erxcCBXvP7sw2IzcDsfecdbX3mGaXcc4/SH3tMtrhYOdu3qygvr8p5DixcqDM7dyr1vvsUl5amvCNHtPO//1srb7+95PiRnOxV3+lxWYb91Flq+1zvS306XrCvOoc/57JQis3yDsyfr5xvvqlUHqrbI9hcOR7W2ipfcXFxNlDFRUWev/fOmWPnp6ba3P37veocWrrUzk9NtcfWrvWUFeTk2MVdu9qvxo/3lJ3ets3OT021++bO9ZQVXbhglw0caD978MFq21F04YJd0q2b/WLMGK/yzb/5jV18/fW2qKAg4L7VR24dj4yMDJuRkRHQPL4cW7fOzk9NtUdXr66yjr/bx5e8Y8fswo4d7Tcvv+xVvnboULti8OBLa7xL+ROzecePV5rv+GeflcTne+/VuI6/9+1rN40ademN9aG2YtNXH7/74IOSWFyzxlprbe7+/XZ+aqrdO2dO4MsnNgOSu3+/XXj11fbb//mfgObzNY65WVl2fvv2dvvkyV7ldRmX1tZebFpb//dTt6nNc70vl+N4UVvx6YZ91U38OZeFSmyWV5Cdbf/vhhts1rx5dn5qqt3+4oueaU7fHm5U38dD0kbrI5ev9Xv0y3/LVJXDy5YpqnlzJfXs6SkLj49X85tu0uGPP/aUHfn4Y5nwcCXfequnrEFYmFr9y7/o2KpVKsrPr3IdpzZvVsHJk2p9++1e5a1vv10XTp3SyY0bA+lWvcV4XDp/t48vR1etUnFBgVr95Cde5a1uv11nMjN1bv/+OmlzfeZPzEY2aVKprOyKjLzDh2u9TcFQXR/PHzlyycsnNgPzXek3T20DvNXI1zjGtGqliMRE5dXCOAYL+6mz1Oa53pf6dLxgX3UWf85loRKb5W2bNEnxaWlqddttlaaF4vYINreOR1Aexndm507Fp6dXKo9PS9P5gwdVmJvrqRfTurXCoqMr1SsuKNC5ffuqXYekSuuJT0srmb5r1yX1wU0YD+mLUaO0IC1NS66/Xl+MHKlzBw96pvm7fXw5u3OnGkREKDYlpdK8UvD77SYnPvtMkhR31VV+1T+ybJkWXXONFl19tVbdeacOLV1al82rFSc2bJAkxbdv71X+zYsvamF6uhZfe602PPSQcjIza1wWsRmYkxs3Ki41VQcXLNCyAQO0MD1dywYM0J7ZswNe1pldu1Rw4kSlcZTqZ1wGIhT2U6cKlXMZ+6rzVTyXhUpsljmxcaOy/vY3fe/ZZ31OD7XtEWxuHo9av0ffHxdOn1ZMq1aVysMbNSqZnp2tsNhYFWRnKzwhocp6BdnZ1a5DUqX5PesonY7QHo/w+HilPvCAmvToobC4OOVs26adU6boxJAh6jd/viKTkvzePr4UnD6t8IYNZYzxKo8o3Q4FxGGtKDx7VlsnTFDcVVepxQ9/WGP95jfdpEZduiimTRvlHz+uvbNna+Pw4er60kuVrjpxivOHDyvz5ZeV1Lu359uQBhERavtv/6amffsqIjFRZ7/9VjunTtXqn/1Mff/3fxVfTTJFbAYm7+hR5R89qm3PP6+OY8Yo9sordXDxYm0dP162sFCp99/v13KKCwu15Xe/U0Riotr86796TauPcRmIUNhPnSxUzmXsq87m61wWKrEpScUXLmjL2LFq/8tfKq6KZzqE0vYINrePR1ASfVkrVeiwp7zC+4obxmc9n6uouQ5KhfB4JFxzjRKuucbzPqlHDyV2767Vd9yhPX/+c8kTN/3dPr5UMa9Tt0d9VFxYqE0jRyrvyBH1ee89NQir+bD2vfHjvd63HDRIq+68U9tfeMGR/0krzM3V57/6lUxYmK57/nlPeVSzZuoyYYLnfZPu3dWsXz99Mniwdk6Zou9Pnlz1QonNwBQXq/DsWXWbMkUtb75ZkpTUq5fOZWVp17RpajdsmO/jYwVbx4/XqS++0A0zZ3pO9GXqW1wGIhT2U8cLlXMZ+6pjVXUuC5nYlLRr+nQV5+Up7dFHq64UQtsj2Nw+HkG5dD+8USNd8PHtb1lZ2be+EQkJPj/pKKtX8cBbXkS5T1m85i37Zrl0OhiPihp17qzYdu10+quvJPm/fXwpm7fiDn0hJ0fSP7cLLo4tLtY/fv1rHV+zRt2nTlXDjh0vajnmiiuUPHiw8g4fVt7Ro7XcyktTlJ+vDQ89pHP79+vGN9+s8emu0cnJSrz+ep3esqXaesRmYCIaN5YkNe3d26u8ad++yj9+XPl+xM32F17QvjlzdO2kSWrWt2+N9Z0cl4EIhf20PgiVcxn7qjNVdy4Lldg8d/Cgdk6Zog6jRqm4oEAXcnI87Sx7b4uKQmZ7BFsojEdQEv34tDTPPdvlndm1S9HJyZ7LH+LT03UuK0uF589XqtcgIkIxbdtWuw5JldZTdi9EdZe0hhrGw4dyO6S/28eXqp5fcLbsmQVO63c9s2XsWB1ctEjff/XVSv+pu2h+fNNzuRRfuKCNjzyi01u2qMcbb/j1e61S6SfFNfSD2AxM2TGskrJjRQ0PQ9vxpz9p17Rp6vy736nNT38aeAMcFJeBcvt+Wl+EyrmMfdV5ajqXhUpsnvvuOxXn52vz6NFa0rWr5yWV/Azhkq5dlZOZGTLbI9hCYTyCkug3HzhQeYcP63jpQ3kk6cKZMzqyfLmal/st6OYDB8peuKBDH33kKSsuLNTBRYvUtE8fXREZWeU6GnftqojERB2YN8+r/MC8eQpv1EiJ119fiz2q3xgPb6e3bNHZPXvU+LrrJPm/fXxp1q+fGkREKGv+fK/yrHnzFJ+erpg2bWq/AyHi64kT9d177+na559Xy0GDLmlZxYWFOvjRR4pOTlZU06a11MJLY4uL9cXo0Tq+dq26T5umxqUnn5qcO3hQp774Qo2vvbbaesRmYFqUxtjRCr9PfmzVKkW1aFFt3Ox+801lTp6sjmPGqN199/m9TifGZaDcvp/WJ6FyLmNfdRZ/zmWhEpsJnTqp59tvV3pJJU9h7/n224pt2zZktkewhcJ41Mk9+gcXL5YkZW/dKkk6+umnikhMVERiopJ69FCLH/xAjbt21ebRo9XpqacUnpCgXdOmSdbqqoce8iwnoVMnJd96q76eMEHFhYWKad1a+955R+f271fXCveeLhswQDGtWqnnW29JkhqEh6vDqFH66umnS34SoXdvHV+3Tt/NnavO48apQUREXXTdkRiPqn0xapRiWrdWQufOCo+PV/a2bdo5bZqimjdXyr33SpLf20eSFqanq/Udd+i6SZMkSZFJSUq9/37tmjpVYbGxSrjmGh1ctEjH161T9+nTL3t/64uaYnbX9Ona/cYbavOznykuJUWnNm/2zBuRmKjYcleXVByTA/Pn6/DHH6tZ//6Kbtmy5EFKb72l7K1b9f1XXrmMvazeV+PG6dBHHyntkUcUFhPj1ceoFi0U3bKlvp44Uba4WImlH6Sd3bNHu6ZOlYxR2iOPeC2P2Lw0zfr3V5Mbb9SWsWNVcOqUYtq00aHFi3Vs1SrPvabnDhzQ8gEDlD5ihNJHjJAkHViwQF9PmKCm/fopqWdPr3EMi4vzfPtYX+KyPPZTZ6mtc71Uv48X7KvO4s+5LFRiM7xhQyXdeKPPaTHJyZ5pobI9gi0UxqNOEv1Njz3m9f6rp5+WJDXp0UNJ77wj06CBbpg5U9v+8Ad9NW6civLzldi1q3q+/baik5O95r3uj3/UNy+9pMzJk3UhJ0cNr75aPWbNUqPOnb3q2aIi2eJir7KU0t9Q3f3GG/p25kxFt2yp740fr5Sf/7y2u+xojEfV4tPTdWDBAu35y19UlJenyKQktRw0SB1GjlRkYqIkBbR9bFGRbFGRV1nHMWN0RUyM9rz5pvKPH1dsu3a6/vXX1aKGTwFDWU0xe/TTTyVJ++fO1f65c73qtr7jDnV94QXP+4pjEtOmjfJPnNC2SZN0ITtbV0RFqVGXLuoxa5aa9etXV10KWFkfd06Zop1TpnhNS3/8cXV44gnFp6Vp79tvK+uDD1SYm6uIxo2V1LOn0h9/vNLTY4nNS2OMUfdp0/TNiy8q85VXdCEnR3Gpqer68stqXfa7u9ZWOvYdXblSslbHVq7UsZUrvZbZpEcP9XrnHUn1Jy7LYz91lto819fn4wX7qrP4cy4Lldj0F9vDWerzeJjqnvoXHx9vz5w5U+eNAALVv39/SdInn3wS1HYAFRGbcCpiE05GfMKpiE04nTFmk7W2W8XyoNyjDwAAAAAA6gaJPgAAAAAALkKiDwAAAACAi5DoAwAAAADgIiT6AAAAAAC4CIk+AAAAAAAuQqIPAAAAAICLkOgDAAAAAOAiJPoAAAAAALgIiT4AAAAAAC5Cog8AAAAAgIuQ6AMAAAAA4CIk+gAAAAAAuAiJPgAAAAAALkKiDwAAAACAi5DoAwAAAADgIiT6AAAAAAC4CIk+AAAAAAAuQqIPAAAAAICLkOgDAAAAAOAiJPoAAAAAALgIiT4AAAAAAC5Cog8AAAAAgIuQ6AMAAAAA4CIk+n56//33NWLECPXt21cNGzaUMUY///nPq51n7dq1+tGPfqTExETFxMSoS5cueuWVV1RUVFTlPH/+8591ww03KC4uTgkJCerfv78WLlxY290BlJWVpV/84hdKTk5WZGSkUlJSNHLkSJ06dSrYTQOITzgWsQknIz7hJLNnz5YxRsYYzZw502edi8mX4J+wYDegvpgwYYK+/PJLxcXFqXXr1vrmm2+qrT9v3jzdeeedioqK0l133aXExEQtWLBAo0aN0po1azR37txK8zz55JN66aWX1Lp1az344IMqKCjQnDlz9OMf/1ivv/66HnvssbrqHkLMt99+q169euno0aP6yU9+oo4dO2rDhg169dVXtWTJEq1Zs0ZNmjQJdjMRoohPOBWxCScjPuEk+/fv14gRIxQXF6ezZ8/6rHMx+RICYK2t8hUXF2dRYvny5XbHjh22uLjYrlixwkqyQ4cO9Vk3OzvbNm3a1EZERNjPP//cU37+/Hnbs2dPK8m+++67XvOsWbPGSrLt27e3J0+e9JTv2bPHJiYm2sjISLtnz5466Vt9lJGRYTMyMoLdjHpr0KBBVpJ97bXXvMpHjRplJdlf/epXQWpZ/UdsXjris24Qm5eO2Kw7xOelIz7rBrEZuOLiYjtw4ECbmppqn3zySSvJzpgxw6vOxeRL8E3SRusjl+fSfT8NGDBAaWlpMsbUWPf999/XsWPHdPfdd6tbt26e8qioKE2YMEGSNHXqVK95pk2bJkn6r//6LzVu3NhTnpKSokcffVT5+fmaNWtWbXQFIW737t1aunSpJ7bKe+aZZxQbG6vZs2crNzc3SC1EKCM+4VTEJpyM+ISTvPbaa1q+fLlmzZql2NhYn3UuJl9CYEj068Dy5cslSbfcckulaf369VNMTIzWrl2r/Px8v+YZPHiwVx3gUpTF0aBBg9SggfchID4+Xr1799a5c+e0fv36YDQPIY74hFMRm3Ay4hNOsX37dj311FN64okn1K9fvyrrXUy+hMCQ6NeBzMxMSVJ6enqlaWFhYWrXrp0KCwu1e/duSVJubq4OHDiguLg4tWzZstI8aWlpkqQdO3bUYasRKqqLT4l4Q3ARn3AqYhNORnzCCQoLC3XPPffoyiuv1MSJE6utG2i+hMDxML46kJ2dLUlKSEjwOb2s/PTp0xdVH7gUxBucjPiEUxGbcDLiE07w7LPPavPmzVq9erWio6OrrUvM1j2+0Q+CkmcmyK/7/csLtD5wMS42PoHLgfiEUxGbcDLiE3Vtw4YNmjhxosaMGaOePXte8vKI2UtHol8Hyj6BKvukqqKcnByvejXVr+kTLyAQgcYncDkRn3AqYhNORnwimMou2U9PT9fvf/97v+YhZuseiX4d6NChgyTf90EVFhZqz549CgsLU2pqqiQpNjZWrVq10tmzZ3Xo0KFK8+zcuVNS1fddAYGoLj4l4g3BRXzCqYhNOBnxiWA6e/asduzYoe3btysqKkrGGM/rmWeekSQ9+OCDMsZo5MiRkgLPlxA4Ev06cNNNN0mSlixZUmnaypUrde7cOfXq1UuRkZF+zbN48WKvOsClGDBggCRp6dKlKi4u9pp25swZrVmzRtHR0brxxhuD0TyEOOITTkVswsmITwRTZGSkHnjgAZ+vrl27SpL69OmjBx54wHNZ/8XkSwgMiX4dGDJkiJKSkjRnzhxt3LjRU56Xl6exY8dKkoYPH+41z8MPPyxJeu6553Tq1ClP+d69e/WnP/1JkZGRuv/++y9D6+F27du316BBgzyxVd64ceOUm5ure++9t8rfPQXqEvEJpyI24WTEJ4IpOjpaM2fO9Pm67bbbJEn33XefZs6cqbvuukvSxeVLCAxP3ffThx9+qA8//FCSdPjwYUnSunXrNGzYMElSUlKSXnzxRUlSw4YNNWPGDA0ZMkT9+/fX3XffrcTERM2fP1+ZmZkaMmSIJ8jL9OrVS6NHj9bkyZPVpUsXDRkyRAUFBfrrX/+qkydP6vXXX1dKSspl6y/cbcqUKerVq5cef/xxLVu2TFdffbU+++wzrVixQunp6XruueeC3USEMOITTkVswsmIT9QnF5MvIUDW2ipfcXFxFiXGjRtnJVX5atu2baV5Vq9ebQcPHmwbNWpko6KibOfOne3kyZNtYWFhlet58803bbdu3WxMTIyNi4uz/fr1swsWLKjDntVPGRkZNiMjI9jNqNe+++47O2zYMNuiRQsbHh5ur7zySvv444/bEydOBLtp9RqxWTuIz9pHbNYOYrNuEJ+1g/isfcTmpSnLoWbMmOFz+sXkS/AmaaP1kcsbW/rTBb7Ex8fbM2fO1P2nDUCA+vfvL0n65JNPgtoOoCJiE05FbMLJiE84FbEJpzPGbLLWdqtYzj36AAAAAAC4CIk+AAAAAAAuQqIPAAAAAICLkOgDAAAAAOAiJPoAAAAAALgIiT4AAAAAAC5Cog8AAAAAgIuQ6AMAAAAA4CIk+gAAAAAAuAiJPgAAAAAALkKiDwAAAACAi5DoAwAAAADgIiT6AAAAAAC4CIk+AAAAAAAuQqIPAAAAAICLkOgDAAAAAOAiJPoAAAAAALgIiT4AAAAAAC5Cog8AAAAAgIuQ6AMAAAAA4CIk+gAAAAAAuIix1lY90ZhjkvZdvuYAAAAAAAA/tbXWNq1YWG2iDwAAAAAA6hcu3QcAAAAAwEVI9AEAAAAAcBESfQAAAAAAXIREHwAAAAAAFyHRBwAAAADARf4foEfbYGOfOmYAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 1296x144 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "generate_visualization(terminal_left_reward, terminal_right_reward, each_step_reward, gamma, misstep_prob)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}