Thor DQN Atari Paper.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Open AI Gym"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "    <video width=\"360\" height=\"auto\" alt=\"test\" controls><source src=\"data:video/mp4;base64,AAAAIGZ0eXBpc29tAAACAGlzb21pc28yYXZjMW1wNDEAAAAIZnJlZQAALeFtZGF0AAACrgYF//+q3EXpvebZSLeWLNgg2SPu73gyNjQgLSBjb3JlIDE1NSByMjkxNyAwYTg0ZDk4IC0gSC4yNjQvTVBFRy00IEFWQyBjb2RlYyAtIENvcHlsZWZ0IDIwMDMtMjAxOCAtIGh0dHA6Ly93d3cudmlkZW9sYW4ub3JnL3gyNjQuaHRtbCAtIG9wdGlvbnM6IGNhYmFjPTEgcmVmPTMgZGVibG9jaz0xOjA6MCBhbmFseXNlPTB4MzoweDExMyBtZT1oZXggc3VibWU9NyBwc3k9MSBwc3lfcmQ9MS4wMDowLjAwIG1peGVkX3JlZj0xIG1lX3JhbmdlPTE2IGNocm9tYV9tZT0xIHRyZWxsaXM9MSA4eDhkY3Q9MSBjcW09MCBkZWFkem9uZT0yMSwxMSBmYXN0X3Bza2lwPTEgY2hyb21hX3FwX29mZnNldD0tMiB0aHJlYWRzPTcgbG9va2FoZWFkX3RocmVhZHM9MSBzbGljZWRfdGhyZWFkcz0wIG5yPTAgZGVjaW1hdGU9MSBpbnRlcmxhY2VkPTAgYmx1cmF5X2NvbXBhdD0wIGNvbnN0cmFpbmVkX2ludHJhPTAgYmZyYW1lcz0zIGJfcHlyYW1pZD0yIGJfYWRhcHQ9MSBiX2JpYXM9MCBkaXJlY3Q9MSB3ZWlnaHRiPTEgb3Blbl9nb3A9MCB3ZWlnaHRwPTIga2V5aW50PTI1MCBrZXlpbnRfbWluPTI1IHNjZW5lY3V0PTQwIGludHJhX3JlZnJlc2g9MCByY19sb29rYWhlYWQ9NDAgcmM9Y3JmIG1idHJlZT0xIGNyZj0yMy4wIHFjb21wPTAuNjAgcXBtaW49MCBxcG1heD02OSBxcHN0ZXA9NCBpcF9yYXRpbz0xLjQwIGFxPTE6MS4wMACAAAACAmWIhAA3//728P4FNlYEUGa7Q91nCgDAQZ/NTMClgclA4pYaytdZh+dVJuV432kRnWAs5rNIwVgDq82rBm7oYytwVrbMx0s0deIjDqA1k01uG4ANrMfO1grOJwZga71GmymuARhsR4WZ3pZaGRcKmQ1AKWNgP+g0jF9vZyFT5gpot86JTbMFc0ZcdazcAe48r5fEfvVute9rB6tUebo7LSDmgk0Yk5SccnTIC7OKXa0z9mVrKEExQtVw/h9gGhuwMIFSbYTBIps6sOwginkaQahoy3YpigJeHW1QeapzVRURgWt3Cb/fABnsQJUbQBnzSOFXARJncJs9Egfvxb8tPmLx1saqciKlnYZZ3TBjw5BDKy2pLMIQlUrsVKYTYX9hV+lrvVOAfbYkUg1VgiZyygVNBBkK0cr2V9Z9KA4o9WtvG9L3J2bT1I7ByLycHnVVT6CxO8EXVlpG+CGjAAHXgWgnJa5SvEPAl+X3OxVbIk1DOPs14GLO4QLTtWDjk9HtX3mTv429/gyyrSknj7YaLkwBvHDLaGLQS2qJ7RJ41b/fvlWWrnE6YaWbaw12m2gQjRs0if+Mecx1jTnogqPN/IX2LkuCBcdUtmybks7eciFuIHKpSsh2JpI/fSFz8cPzETS+wQpr5OlqLNMfCoZakM2bwcbZNqudywUIjmSRe94IU1kAAACPQZokbEN//qeEAxdTdIBabod563XnQ29qI2wZ9BJPlNgTpOuQYQVfk77dXt7/6Nu2tTcxt7uC5EMisi/ew/aEIz9DQ/xXhf/tD7UF2jlVAotJQuE170NMQEKTCNbhVbJ6ar/YiOaVaW/A94pXYiJLMnCGsHG8Tw5XSeMwL3exOjejQ2DkImAL2z/ZaZ7LqkwAAAAmQZ5CeIV/AbE6zb6fgUOnbo9xACFO6nramZNS2cmTJ/7paA9QgE0AAAAVAZ5hdEJ/Abfx1VYIHkQtdbDdWx9QAAAAKAGeY2pCfwBh+/gZ1iEntiQAEPlxe4wR2XTSQoIVux6NW0qcusHNta8AAABjQZpoSahBaJlMCGf//p4QCv9kNqOJkv+BxUwAXOHhHXvrEZ7QXJaJWFS8MAGMlAQK6qLeY1Rp6f1+T34gHwRY5/bOUUWB7zL8h7My7XU3Dex7Sqo1l3SIu1MN7WgSLJeK+FzBAAAAMEGehkURLCv/Aacf8fEVSH+JkMycjlrDEA1fOjsZj6HrfR3KZz0rNM/IAy4FBDIE5wAAABABnqV0Qn8CGNJ6oT3pOPG1AAAALwGep2pCfwEtk8qAIcIyNU+lWd1PeRxguTY8a78TAETkwnyy3RxpUzND+MQjm6/FAAAAN0GaqUmoQWyZTAhn//6eEAilf/8DkJgAueYi3eiygN85SS/2BFBkijPw4n32dvLtokSA2MsyTOAAAAAxQZrKSeEKUmUwIb/+p4QCk95r3IbFz8ABsOJ/Pte6lstqdO9ijJUayP/5kEbjGe+TgQAAAFBBmu1J4Q6JlMCG//6nhAIL6DbyLihtmnbhBegk8TDMr1i1wAZ8NuZZhsog8jOHKS91KN/rYy98apJAGEV3kRNoy9/ay83ig9rop2zc3XaNcAAAABtBnwtFETwr/wFsaJ9B6PYS8veQpLjaYX0CwcAAAAAOAZ8sakJ/AO2EvkDhyj0AAAA5QZsxSahBaJlMCG///qeEALF7wpvwHPJmqSNFa6cIwft2vjoAjFneHowC/uFcecDAiRrmxFJweiU/AAAANEGfT0URLCv/AJK8aaIsS55vsxW6IwVgAj3nR2JizCVLYITd9tyrmaZQgagb3DpA9fdTiNcAAAAhAZ9udEJ/AL7vteJX/RZtMoTXjcaAwAEQddmkA6SNUazoAAAAFgGfcGpCfwC/Cq9o4c9nrQFa9uwvvhYAAABNQZt1SahBbJlMCG///qeEAIt8jZyGV9T9mfwgBOtazXpOsxdX0K0aWHoeD9ofjhoTzIcz6fhhYnrOfIZybFDW0X18XZxzgDlmMeOnES8AAAA4QZ+TRRUsK/8AdFQ9QA15a2e+cn6fElxZIroIK3/8j1wX9cZE0FTjOULlZINsAIBnzeERI0LokCgAAAAWAZ+ydEJ/AJbu7p1KNXaKaNC4btQLSAAAABYBn7RqQn8AlsRiO2zZ5WXoZnS2bwtFAAAAGUGbuUmoQWyZTAhv//6nhABu/UZVhHP1yUEAAAArQZ/XRRUsK/8AXXxIAAHLNVekF8VB7WcZnybR7nAqoOqLM3+g6Bvx/1hWKQAAAB8Bn/Z0Qn8Adux+YADYsJeoz41pAm94Gsc7bb9700HjAAAAGwGf+GpCfwB2y1cn6A1nZc8gBX+hY0oN9e3Q9AAAABJBm/1JqEFsmUwIb//+p4QANSEAAAAOQZ4bRRUsK/8ASpVJ4EAAAAALAZ46dEJ/AGH8/PkAAAALAZ48akJ/AGIeTPkAAABDQZohSahBbJlMCG///pKxUAA6EL//CD1QJM6Ly2OAJibmyYs0C78G0djAC08aWA923NAK9m7JxSDsVbFnb7n5skDEPgAAACJBnl9FFSwr/zlV766TYWAG5ZL7y8cAYVwfsVZNOneprP4wAAAAEgGefnRCf0AZ4fQ8Vz+G882uCQAAAB4BnmBqQn8BJYrkT8yAEU1H00NmKUzQFmkDYxPjssEAAAAnQZplSahBbJlMCG///qeEANf7KkkBEEMAITZl8MccA1+4r82SBd/xAAAAI0Geg0UVLCv/ALE0sUxABxUq+ihswZEc3mL6BQLi/LD9kcWtAAAAIAGeonRCfwDicVuADgQT3544GPlSqoH5VXn8s3OnaEabAAAAHgGepGpCfwDdCsV++0AQ7UfTQ2YpTNAWaQNmBb+ywQAAACdBmqlJqEFsmUwIb//+p4QAo/vCmivrlwAbR1H3A44Br9xX5skC81kAAAAjQZ7HRRUsK/8AgsixTEAHFSr6KGzBkRzeYvoFAuMbpfskSwsAAAAgAZ7mdEJ/AKyltcAHAgnvzxwMfKlVQPyqvP5dC1+oXO0AAAAeAZ7oakJ/AKgzcifmQAimo+mhsxSmaAs0gbUY/6RhAAAAKUGa7UmoQWyZTAhv//6nhAB5/ZUkgIghgBbnUfcA58La/cV+zWWVegDhAAAAIEGfC0UVLCv/AGSINqYgA4qVfRQ2YMiObzF9AnHi5wutAAAAHgGfKnRCfwB++K3ABwIJ788cDHypVUD8qry3v2gbFwAAABsBnyxqQn8AfEJs82ZACKaj6aGzFKZoCzSBsx0AAAAYQZsvSahBbJlMFEw3//6nhABc8Kx1SORhAAAAEQGfTmpCfwBh/4wcAPjV82R/AAAANkGbU0nhClJlMCG//qeEAuAe/AJ8TFBPN61ChzUzgWuxBzacdMUe5TR//3+XqxA+BE77tPZ8dAAAABNBn3FFNEwr/wBknUNZJ4j6vrmAAAAAHQGfkHRCfwBiEeLMAEZCmHqQVffPgMBGFXKmYhUDAAAAHQGfkmpCfwB/H4rgAcXC95cFVQLgMAn+yRfnr3uAAAAAJUGbl0moQWiZTAhv//6nhACnbS1ACDaj/9/l6zVM4RP1GE9RN6AAAAAgQZ+1RREsK/8Ahuwm17MACs2P/3+X8P4JiQc3xQueil0AAAAfAZ/UdEJ/AIMD2MAFh2b3mjBPY7JOpM/PhQgPcEWW0wAAABwBn9ZqQn8AsVvYKvACL8b3lwVVAuAwCgLfwrXJAAAAOkGb20moQWyZTAhv//6nhALf8guLjo60zqN+ftBo+fHx93ZBudEblI7hKgqos//f5d7U4NIwotp7c/sAAAAfQZ/5RRUsK/8AtdgmpPlCAGmE//v8v4ejWZ6gmN+u2wAAAB0Bnhh0Qn8AsUhxngCJhTD1IKvvnwGAjCrlTMQnmwAAABsBnhpqQn8A6D8VwAOLhe8uCqoFwGAT/ZJU3AEAAAAvQZofSahBbJlMCG///qeEAuAtZllwksoO1UvXCJNAA2FH//f5es1TOEPmsJ4/JEEAAAAgQZ49RRUsK/8A8rLNl7MACs2P/3+X8P4JiQc3t8G/2bcAAAAfAZ5cdEJ/AO3VlMAFh2b3mjBPY7JOpM/PhQgPcEWV/wAAABwBnl5qQn8BPduwVeAEX43vLgqqBcBgFAW/hWJIAAAAJkGaQ0moQWyZTAhv//6nhAGuPYFAC1Uf///xgK5IqHZzQe03tEklAAAAIEGeYUUVLCv/AUiwTUnyhADTCf/3+X8PRrM9ObSz0aWAAAAAHQGegHRCfwE9tOM8ARMKYepBV98+AwEYVcqZiEyfAAAAGwGegmpCfwGkfiuABxcL3lwVVAuAwCf7JKm2fQAAAD5BmodJqEFsmUwIb//+GRfpEasFd33y43dASdoNKDyoAJIwAAYmjrHMHz2mzWrIqRhr4u4JyQGxFtDiARwEsQAAACFBnqVFFSwr/1elwP2jjLtb9ugAuHb/g4hve1ecNmI/Zd0AAAAYAZ7EdEJ/Aa6z53ACFNm9nfLvOpixbRXvAAAAIwGexmpCf1/04fskKEAB/Oz7KlFgKt+LcXfale4CfqkmiwLBAAAAhUGay0moQWyZTAhv//6nhAGyErLA0ws+ADVqF1jHXTGdvFJlGYjOPuDlTrWmkpTC8ZsAxOQAAyN3qs+NQ2cyUYwxXX3Br3Lprzi1/piNePeR6P097YQu2v0zWzPhCyuZIFu8SXgAHZgT3ve8l5agB5jadIHYEOM/eYdEjIkMGikDGISfPXAAAAArQZ7pRRUsK/8BSHI+gBaIw5mVkkrT6OM/bKeAlaxMoVD7U4ObfYegEwTBwAAAACUBnwh0Qn8BpEVpIARgJ9lSpGNxsOdRj4OopdV6+kPrWF6IaDj7AAAAMgGfCmpCfwGkFEsAgBYez7KlSMa/xVhpLFoCglbc6jHwcyZpaQTiWRqrbnyFfDirnYE4AAAAN0GbDUmoQWyZTBRMN//+p4QBLfkWm6BgBCBAusY66Yx3xSZRmH85Ikr+ORs2D70TU55FY+9QkzAAAAArAZ8sakJ/AT2JSh4IARgJ9lSpGNxsOdRj4OopYYSmqRLV2HYMC0GuuhpoeQAAADlBmzBJ4QpSZTAhv/6nhAEl+CFDHorREAEPz3UYkAf+3bxf1wFLg8lnqNFUao1tYOuRNDapyqBEKmEAAABzQZ9ORTRMK/8A7QaYwBExmHHBqMa/myZq4N2zG+e8j8HMmXbpagkRMay2+BmbA17aaxU66ximxFg6fyLSAnupO2TRXF5+cBYybFPVTBFZ4XLx0AFv9AngqKDxlXFJxy/k9jmOAHe2fvWXHXJWCralN6HxyQAAAC4Bn29qQn8BNYjEztSxngCEN9oegSStsOb2Q2VBVMzwiElgjm8zAt4/2iDdlJ9VAAAAUUGbdEmoQWiZTAhv//6nhADc+ypIvC4QwAhOe6jEgD/27eL+uApcHfNZV0uXQK2xM3/wAj59AuAABifTYO8yCF4QJkKIrJ9nMxfljslMTpp1XAAAADRBn5JFESwr/wC1tLFMQAjIzDjg1GNf4xuVcHAOiV0d5H4OZM83S1BwcuuLwy8PffYbkbJTAAAALAGfsXRCfwDn8VuADgrNBajGTDpi5fkeD1K1JsHeHy0/y0G1hjQ20fBLeSiAAAAAJwGfs2pCfwDihMazcAG0n2h6BJK2w5vZDZUFV//8ISWBnm3/y5dtEgAAAFJBm7hJqEFsmUwIZ//+nhACffFjYVXvwN/WAF0Eq2X3ZT/2fcaMfCfW60rVsfffgYUfKUCsRS/Inb8qWGfv7vsXp4drMmJ2vdpV6i4ETGmeaVqpAAAANkGf1kUVLCv/AIb6rwxYAiYzDjg1GNfzZM1cG9XIXz3kfg5ky7dLUEYOXmeku1+bpHZr5PhTgAAAACkBn/V0Qn8AsSK4LZwAcFZoLUYyYdMXL8jwepkbJ+9rR6VK1fEwf1kJ1QAAADABn/dqQn8AsVuMMIE8DiEAIc32h6BJK2w5vZDZUFUzPCISWCMa/evihtFduXNrqsEAAAAkQZv5SahBbJlMCG///qeEAHlBjyO6gXSbnvuBE8X3dT7Zl9SgAAAAOEGaG0nhClJlMFFSw3/+p4QAef2VIY6uxoMAIGN1M1UsLZ28UmUZiM4+4OVOtaLcjSfzA9s2Jxt5AAAAMgGeOmpCfwCvStOZRrqngCw9n2VKkY1/irDSWLNFv24lxe6LvtHIuxkL/tLrJ+Fjq7RYAAAAPUGaPknhDomUwIb//qeEAF9ilqAFvXEl09wfmX2wtR+cA6nO1AUoZ3suZ8WqOZ5rg4QHzMxGgkAoDSVp3eEAAAArQZ5cRRU8K/8AhSqUBkbW6rKnVLsAIfjhxwXYDJsvGCK/3LDBbw93rzGaUwAAACUBnn1qQn8AsVwsVREFG/mnDGPACZZ/aKaG8Nb7k+iwu1DJ+FXAAAAAR0GaYkmoQWiZTAhv//6nhAB5SRfABD9DS0IFKb1SH/ZV/kzkYsjOTq4tZ8xaor1rWFhPsHNmRL8nt1nwNark6crNd0rhm23AAAAAMkGegEURLCv/AIcGcwj9nqADj16Msx0aeg2jfIZT7WfBIxO8RCfy/D+kzqEqTYiY2DWBAAAAIwGev3RCfwCxI2L0Ew0tXLkNwAtSPTJ5yAwtqxIxbGjYQEw8AAAAIgGeoWpCfwCxXDQvbjjrwAhcFuURhZ7S2sJEhOAOOEbQPyEAAAA8QZqkSahBbJlMFEw3//6nhACi7ckAC6rTfaI5RkRENYX6ug6zZ7VVG5xiyFp8e7GS+5trLbha7jbtPnhbAAAAIwGew2pCfwCxeak8qLcUun4ARRHplsZMNgtrCRH4SLf9iFaRAAAAO0GayEnhClJlMCG//qeEANbHH4AP6O0LQgTo/sT9XQnZpNpCpBRBpW0dr1o1nzJSbOM38ki/IBoCErDRAAAANUGe5kU0TCv/ALFYDc4vS5ADcz0ZZjp1p+mvyGU1FdDKwn7vAqGloVDHKZTxTMKGG7+kT8B3AAAAIQGfBXRCfwCvSsxvNwANdi3KIws9pbWEiQm+YCAW6+krsQAAAB4BnwdqQn8A4rx/Uq8AIoj0y2MmGwW1hIj8Lhi68ygAAAA/QZsMSahBaJlMCG///qeEAr/YWdMrc5AC6rTfaI5RkRCg0V6ug6zZ7T2n7INKhdogVjfc21ltwtST9i0RIh/AAAAANUGfKkURLCv/AO0ySsx58rQAcWRUNMdOtP01+QymoroZWE/d4FQ0tCoY5Vp3v8bUfOrHadLZAAAAHgGfSXRCfwDoS7hfOABrsW5RGFntLawkSE4A8kbFDAAAACABn0tqQn8BNdtakupBfuAIcj0y2MmGwW1hIj8JH/otuAAAADxBm1BJqEFsmUwIb//+p4QBnbx+AD+jtC0IE6P7E/V0J2aTaQNwUQaVtHa9aNZ8yUmygdVk8Xn5oIcwqCEAAAA1QZ9uRRUsK/8BP7AZHIZcgBuZ6Msx060/TX5DKaiuhlYT93gVDS0KhjlMp4pmFDDd/SJ9+5kAAAAgAZ+NdEJ/AT32JtXAA12LcojCz2ltYSJCb5fp4uTHH7UAAAAeAZ+PakJ/AZp5BVduABv5LctjJhsFtYSI/C4estxwAAAATEGblEmoQWyZTAhn//6eEAt/wYsAR7yWWhbKjpABCuz0ZDiePFUWGhmnvwoxGroyRWNUEMUYcgU9X1Ri6kbnW7DrLCET1MqbMpdHQaAAAAA5QZ+yRRUsK/8BsX+Ulh+6kANzPRlmOnWn6a/IZTUV0MrCfu8CoaWhUMcq073+NqPmXYrBpZR6XM+hAAAAIQGf0XRCfwGkl3C+cADXYtyiMLPaW1hIkJwB5I2JjM+mAQAAACQBn9NqQn8CKdADaFd4OABv5LctjJhsFtYSI/CR0TQWP+oOQcAAAAClQZvVSahBbJlMCG///qeEAxa9XXAFqS5/SnO5k20FhEoYp5AL4DLJsk0voSBp76+d9GWEm1V8DPAXnCHrMMPKSBlBAlsc4q8Cz1Pug8MRgp5MXke3L3Vw7vqOf9FVrnRlt6HWDpzKG5N4SXD1p6uRx4ny5G892DC/BvDL1xx1tMSyI1Fqlz9zFYJ6Wb+wX4/e16LML8uJ+0GaNsAEbu5iJ8+WhQXBAAAAhkGb+UnhClJlMCG//hFUlKVQCEZ+h/mMItwsxDXzGx9qf1xs58qeY8ySGx7/wcMIRSyKyYfiLcvYLc0cbugJMfnwQ+RbSF0Ly+orQ2WRzpcSfckO3i1G5KxnYpwFfJAbCVKbHhIV33iQ5keaojqynqvyhMzNigqhN7dgpLZfqJnfG+tsjBToAAAAJkGeF0U0TCv/Vv2B+yl7eXgAh+dY4Zp/SvkCgoQDkwySTjQD64iHAAAAJQGeNnRCf2BTT/MHbVgAbUcnQMqZ6iAlUmW+LtiwHZgdsGfzxBcAAAAfAZ44akJ/AhkVgS6poAg4b+KUcEdrVpjORLWoFq1vgAAAACdBmj1JqEFomUwIZ//+nhAGeDz2tJONAATtuV9j+N2IRtqAEGCFyIEAAAAfQZ5bRREsK/8BSGlXYgA4qVfRQ2YMbWwfF9Ag/rP4oAAAAB0Bnnp0Qn8Bo/LpbOADgQT8Uo4GQA2ohfYiw/YNYQAAABwBnnxqQn8BmhVnMWjeAIOG/ilHBLVZypIyh2OPAAAAMkGafkmoQWyZTAhv//6nhAEkUKBAAht5PvqjgZio/N6JEclp/ZRfHhoNevlfZmqLMWSrAAAAMEGagknhClJlMCG//qeEASX5GzP7t8cAIbeT76o4GZI1fXdZRQkWmLQipcEotKEpLQAAACVBnqBFNEwr/wDtAT2cbUSAGrvfrnT48P4JhjNvCjPBNJr8p+WBAAAAIQGe33RCfwE1aUNgEAIvu33544AwrdGxSKEWoK+UuxHUowAAAB0BnsFqQn8BLLprSbuADjySe/PG60AA6i4G8wHFoQAAAC9BmsRJqEFomUwU8N/+p4QA1/sqSQFPb8AIEcvyKjglwc8HXNok7kLjMRAuwRfr0wAAACQBnuNqQn8BKk33VSNcAHFt33544Avp1xCzZ2LnOc3vzYrooGkAAAAtQZrnSeEKUmUwIZ/+nhACeo/R9MAAnVP/sfxwC5/KZNfNQ4IjRxQ5/WYeydPBAAAALUGfBUU0TCv/AKzyar+BYAbrXf3l43WWay+N7pTTvnrhUhaU8r93EndfmJ7JQQAAACIBnyZqQn8ArLgzgA4EE/FKOBj5UqpaRqPAP2xOcP90tDN3AAAALUGbKkmoQWiZTAhn//6eEAJt8WNhGj0sAH4D/7wo4GbwzSgC9i6ngwg2UMwyMQAAACZBn0hFESwr/wB/APQoiADaav1zp8eH8Ewxm3hXklxkJhIo+25aoAAAACABn2lqQn8AqDNmATQBEwhPpobKMFd1aeNnwDs30lYQEQAAAGtBm2tJqEFsmUwIZ//+nhABz0MBwATqn/2P44Bb6psmvmoBL++Cbi+QlS/HUT3VG4mPFoUpcchQYi+EJjpBBpGFo3awVGChZ+78sldjT9/+A6cezm3VFQo8Htg3BaCjfh7ct4eAMhvnzj2V4AAAAIZBm41J4QpSZTBRUsM//p4QAdH2PLeeB4A5h9wtFq9RrFRhb6aUqadctVHsegtlVH6w4+ACbTPZASntHE4H9Uk4WH4sQGayw7cQN048IIn3s5YP4VyHHP/MiqoXvqHitKqV15fdTun9dql4Tko9+l0EifAl67wtDkYsEwrCVF+WPVRZi1d4pwAAAGcBn6xqQn8AfELLE9SWuLZxwgALoK3mPySUq5eB00jy1r7aNJaKdzI/Y0l3diIk62Ry8BziOQXIKY8mkwcwmF+rBgnG6iY+s9wMVjP9ddTGkpxePhmeel7pWSNS732X6WFh0bnevhzlAAAAJEGbsUnhDomUwIV//jhABmz3ryMAQIOl+6yHtDyG8mNDdEiYTwAAACdBn89FFTwr/wBYqfZQAE8/UCLvnKxHLBLv/+/yhNrzH+ZybLFqBIEAAAAhAZ/udEJ/AHFPoNhHpBhKbQAQeaGt8zgor7pPDYH4+6eAAAAAKwGf8GpCfwBxblncEJFtOo1hsObkAEaqniDZhUG7fheja397FuXhZpu2MFsAAAAjQZv1SahBaJlMCGf//qEOnsyc0DnSwibT25hrhmadFY4jEhkAAAASQZ4TRREsK/8AWuvcqNLteOn4AAAACQGeMnRCfwA3oAAAABEBnjRqQn8Ac/d7CEL7luCloQAAACFBmjZJqEFsmUwIZ//+nhAJp4bhsDv9pfGrA0Es8ymBYkAAAACMQZpZSeEKUmUwIZ/+gflvitAFq4f/+Hd9AlVaPT86qYNJ73i5BQXq2Cy5RAEm6t0NwVWBsY8e12I7TskWAc3D5SqUBmKP+StJ8PXlbMIW8FwHFn9abFv9eUFR8txptUpNGBe+qI0c0nL+3NWUL/H4oqrF38kbWkfaD9wdblBMLAQq562WGXj55C+xvG0AAAAZQZ53RTRMK/85Ve+R3CeAvrVFZlQMbZKbwQAAABQBnphqQn8AcUGkABeCTFs5A4leWAAAAB9BmppJqEFomUwIZ//+nhABk/Y0aTU6bOSlipZv/2+BAAAAIEGau0nhClJlMCG//qeEAGT9k9KClIEDoU4LBg08SIWAAAAAKEGa3EnhDomUwIb//qeEAGJ4TyU8B4QAHc8815D237pW++y83+3Mq8EAAABAQZrgSeEPJlMCG//+p4QA4mvfgi39wAmZEzlJOsxdX2KrMgkO7uOPJFR4yZnPpUTvCXRJ5X01SWqNnNt2TbKaxwAAAEBBnx5FETwr/wC6VvFWsBiC/mAHBmImz8EgMEOyD8fKVJDtpPlikICpVBQ7B9saV/rnSwqf6txHEd75xTTaeekwAAAANAGfPXRCfwEukXiAKO9FHtFKs7qe8jjBcmx411VfUONwms5S5y3RxpZxGlcUoYfOLT+eC4AAAAAaAZ8/akJ/AO27/lekhsk5GaDY3brHXSlDyYEAAABHQZsjSahBaJlMCG///qeEAOJ7KfgvF5fU/ZnXAAfwfM5STrMXV9CtGlh6Hg/aH44aE8yHM+iySwqNC0K07z5vvgpaijTZd64AAAA7QZ9BRREsK/8AujlniAIIcWz3zk/T4kuLJFdBBW//kevC8J8RNBTdlhER3NJ68mf2bpnj9+uj17ROND8AAAAbAZ9iakJ/AO2DSpsCiHckZT0Ii7lRXFXmkHgsAAAAP0GbZ0moQWyZTAhv//6nhACxe0V8J96CX1P2aJYAD+D5nKSdZi6voVo0sPQ8H7Q/HDQnmQ5n0uS3nkDSqgvmuQAAADVBn4VFFSwr/wCTPkAAQo4tnvnJ+nxJcWSK6CCt//SvCB7ZE0H8ucoXKyQbYAQFI71BvEGvXQAAABgBn6R0Qn8AvzuhGat3gPQZk6oLR4kjK8EAAAATAZ+makJ/AL8Jkjts2eXTXzchhwAAADpBm6tJqEFsmUwIb//+p4QAiq2mThUHM17p78RwgBOtazXpOsxdX0K0aWHoeD9ofjhoTyXlM+V4gP2cAAAAMkGfyUUVLCv/AHRRLAAhRxbPfOT9PiS4skV0EFb/+leED2yJoP5c5QuVkg2wAgKR3p/fAAAAFQGf6HRCfwCW7u6dTCNFEoVytQVTgQAAABEBn+pqQn8AlsRiO2zZ5dNeGwAAABlBm+9JqEFsmUwIb//+p4QAbnAzGUb7eERGAAAAJ0GeDUUVLCv/AF18SAAByzVXpBfFQe1nGZ7m1m5fUszf6DoG+0PdQQAAABwBnix0Qn8Adux+YADYsJeoz41pAm94Gsc6EG1VAAAAGQGeLmpCfwB2y1cehucRABX+hY0oN9e3pMEAAAASQZozSahBbJlMCG///qeEADUgAAAAC0GeUUUVLCv/ACtgAAAACQGecHRCfwA3oQAAAAkBnnJqQn8AN6AAAABVQZp3SahBbJlMCG///pK0vcAcU8f/sMPGtlbWjS/UCxMOicPLv+8cH7AAZE+NtzQDobT52dBY1UMJD/8BcbveNHeV53stqOOgd1SfAhNsX0IMsiq+lgAAABNBnpVFFSwr/zlV768aDgcm2eEzAAAADQGetHRCf0AZ4fYRz+wAAAAOAZ62akJ/AS3bWpLVLfsAAAA7QZq5SahBbJlMFEw3//6nhAEd+Rq3g4AXunvwSwAH8HzOUk6zF1fQrRpYeh4P2h+OGhPMhzPpcHedtkUAAAASAZ7YakJ/AS2IxHbZs8rLbRjTAAAAOUGa3UnhClJlMCG//qeEANz6jKsJA1he6e/CXAAfwfM5STrMXV9CtGlh6Hg/aH44aE8l5TPldttgcQAAADJBnvtFNEwr/wC6sFgAQo4tnvnJ+nxJcWSK6CCt//SvCB7ZE0H8ucoXKyQbYAQFI70++wAAABUBnxp0Qn8A7buhGat3gPQZk6oKFoEAAAARAZ8cakJ/AO2DSpsCiHtkRD0AAAA6QZsBSahBaJlMCG///qeEALCIZjKOTrC909+I4QAnWtZr0nWYur6FaNLD0PB+0Pxw0J5LymfK8QH4OAAAADJBnz9FESwr/wCTPkAAQo4tnvnJ+nxJcWSK6CCt//SvCB7ZE0H8ucoXKyQbYAQFI70/UQAAABQBn150Qn8AvzuhGat3gPQZk6oKVwAAABEBn0BqQn8AvwmSO2zZ5dNdywAAADpBm0VJqEFsmUwIb//+p4QAiq2mThUHM17p78RwgBOtazXpOsxdX0K0aWHoeD9ofjhoT3fF8+V4gP2dAAAAMkGfY0UVLCv/AHRRLAAhRxbPfOT9PiS4skV0EFb/+leED2yJoP5c5QuVkg2wAgKR3p/fAAAAFQGfgnRCfwCW7u6dTCNFEoVytQVTgQAAABEBn4RqQn8AlsRiO2zZ5dNeGwAAABlBm4lJqEFsmUwIb//+p4QAbv2U/BeXLxq5AAAAJ0Gfp0UVLCv/AF18SAAByzVXpBfFQe1nGZ7m1m5fUszf6DoG+0PdQQAAABwBn8Z0Qn8Adux+YADYsJeoz41pAm94Gsc6EG1UAAAAGQGfyGpCfwB2y1cehucRABX+hY0oN9e3pMAAAAASQZvNSahBbJlMCG///qeEADUhAAAAC0Gf60UVLCv/ACtgAAAACQGeCnRCfwA3oAAAAAkBngxqQn8AN6EAAAArQZoRSahBbJlMCG///ppQcOirUhAJsDu/+wwi3H4AIF9r9QQsG79rUAAsoQAAAA1Bni9FFSwr/zlV74i5AAAACwGeTnRCf0AZ4b0IAAAACQGeUGpCfwA3oAAAADRBmlVJqEFsmUwIb//+p4QA3OuyaJJ/a4AD+D5nKSdZi6voVo0sPQ8H7Q/HDQnkvKZ9PUVhAAAALkGec0UVLCv/AOgiWABCji2e+cn6fElxZIroIK3//GG0TQenCwiI7mk5010160AAAAAOAZ6SdEJ/AS32e8RH39AAAAASAZ6UakJ/AS2IxHbZs8rLbRjTAAAAP0GamUmoQWyZTAhv//6nhADb4GYyjk6wvdPfiOEAJ1rWa9J1mLq+hWjSw9DwftD8cNCe74vnyvEB/QfKWgtsegAAADZBnrdFFSwr/wC6sFgAQo4tnvnJ+nxJcWSK6CCt//SvCB7ZE0H8ucoXKyQbYAQFI70/+KMPQYEAAAAWAZ7WdEJ/AO27oRmrd4D0GZOqCsx6KQAAABUBnthqQn8A7YNKmwKIe2RFbdgM3iAAAAA/QZrdSahBbJlMCG///qeEALF7wlvCBrC909+CWAA/g+ZyknWYur6FaNLD0PB+0Pxw0J7vi+fK7bbH56pi5WclAAAANUGe+0UVLCv/AJM+QABCji2e+cn6fElxZIroIK3/9K8IHtkTQfy5yhcrJBtgBAUjvUHKHiXtAAAAGQGfGnRCfwC/O6EZq3eA9BmTqgtIW9ZuusEAAAARAZ8cakJ/AL8Jkjts2eXTXcsAAACaQZsBSahBbJlMCG///qeEAIt8jVvCBrC909+CWAA/g+ZyknWYur6FaNLD0PB+0Pxw0J5kOZ9Lg872JEQvUEPLwLHm6Q8f3MJFLAa0I7PbZIFWkEd7sMDfDldEfuj7apNPLd6zrjFiictRka5prBx4W4OLgz7X/0WGSwMO1SrqICCzc1z7pz9xnvNbcIMsaPXkM7zUwHDexQhRrgAAADpBnz9FFSwr/wB0VD1ADXlrZ75yfp8SXFkiuggrf/yVyDwXB8RNBXkzlC5WSDbACApF3Ao6X/2X3nqsAAAAFQGfXnRCfwCW7u6dTCNFEoVytQVTgQAAABkBn0BqQn8AlsRiO2zZ5fWNHCvQiyROGyrgAAAAKkGbRUmoQWyZTAhn//6eEAGx9RVO0l5zrTwx3mk2yLuLc6pqIqBhmY+HoQAAACxBn2NFFSwr/wBdfEgAAcs1V6QXxUHtZxmfR/UegVlcvqWZv9B0Dd21ZIMxgAAAACEBn4J0Qn8Adux+YADYsJeoz41pAm94Gsc8qrS2/CgJsecAAAAZAZ+EakJ/AHbLVx6G5xEAFf6FjSg317ekwQAAAB9Bm4ZJqEFsmUwIZ//+nhABh183l8alV/HHQAttAuS7AAAAHkGbqEnhClJlMFFSwv/+jLABiqQ/0s4QPpyDg8ZgQQAAAA4Bn8dqQn8AaYTJM1XcYAAAABpBm8lJ4Q6JlMCF//6MsAGUWhI5L1bwvK+QMAAAAB9Bm+tJ4Q8mUwUVPCf//fEADyW9twArqAHH+DxpC2nxAAAADwGeCmpCfwBuhMg3cpwiKAAADdNtb292AAAAbG12aGQAAAAAAAAAAAAAAAAAAAPoAAAeuwABAAABAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAM/XRyYWsAAABcdGtoZAAAAAMAAAAAAAAAAAAAAAEAAAAAAAAeuwAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAEAAAAAAoAAAANIAAAAAACRlZHRzAAAAHGVsc3QAAAAAAAAAAQAAHrsAAAQAAAEAAAAADHVtZGlhAAAAIG1kaGQAAAAAAAAAAAAAAAAAADwAAAHYAFXEAAAAAAAtaGRscgAAAAAAAAAAdmlkZQAAAAAAAAAAAAAAAFZpZGVvSGFuZGxlcgAAAAwgbWluZgAAABR2bWhkAAAAAQAAAAAAAAAAAAAAJGRpbmYAAAAcZHJlZgAAAAAAAAABAAAADHVybCAAAAABAAAL4HN0YmwAAACYc3RzZAAAAAAAAAABAAAAiGF2YzEAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAoADSAEgAAABIAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAY//8AAAAyYXZjQwFkAAz/4QAZZ2QADKzZQod+IhAAAAMAEAAAAwPA8UKZYAEABmjr48siwAAAABhzdHRzAAAAAAAAAAEAAADsAAACAAAAABRzdHNzAAAAAAAAAAEAAAABAAAHIGN0dHMAAAAAAAAA4gAAAAEAAAQAAAAAAQAACgAAAAABAAAEAAAAAAEAAAAAAAAAAQAAAgAAAAABAAAKAAAAAAEAAAQAAAAAAQAAAAAAAAABAAACAAAAAAIAAAQAAAAAAQAACAAAAAACAAACAAAAAAEAAAoAAAAAAQAABAAAAAABAAAAAAAAAAEAAAIAAAAAAQAACgAAAAABAAAEAAAAAAEAAAAAAAAAAQAAAgAAAAABAAAKAAAAAAEAAAQAAAAAAQAAAAAAAAABAAACAAAAAAEAAAoAAAAAAQAABAAAAAABAAAAAAAAAAEAAAIAAAAAAQAACgAAAAABAAAEAAAAAAEAAAAAAAAAAQAAAgAAAAABAAAKAAAAAAEAAAQAAAAAAQAAAAAAAAABAAACAAAAAAEAAAoAAAAAAQAABAAAAAABAAAAAAAAAAEAAAIAAAAAAQAACgAAAAABAAAEAAAAAAEAAAAAAAAAAQAAAgAAAAABAAAGAAAAAAEAAAIAAAAAAQAACgAAAAABAAAEAAAAAAEAAAAAAAAAAQAAAgAAAAABAAAKAAAAAAEAAAQAAAAAAQAAAAAAAAABAAACAAAAAAEAAAoAAAAAAQAABAAAAAABAAAAAAAAAAEAAAIAAAAAAQAACgAAAAABAAAEAAAAAAEAAAAAAAAAAQAAAgAAAAABAAAKAAAAAAEAAAQAAAAAAQAAAAAAAAABAAACAAAAAAEAAAoAAAAAAQAABAAAAAABAAAAAAAAAAEAAAIAAAAAAQAACgAAAAABAAAEAAAAAAEAAAAAAAAAAQAAAgAAAAABAAAGAAAAAAEAAAIAAAAAAQAACAAAAAACAAACAAAAAAEAAAoAAAAAAQAABAAAAAABAAAAAAAAAAEAAAIAAAAAAQAACgAAAAABAAAEAAAAAAEAAAAAAAAAAQAAAgAAAAABAAAEAAAAAAEAAAYAAAAAAQAAAgAAAAABAAAIAAAAAAIAAAIAAAAAAQAACgAAAAABAAAEAAAAAAEAAAAAAAAAAQAAAgAAAAABAAAGAAAAAAEAAAIAAAAAAQAACgAAAAABAAAEAAAAAAEAAAAAAAAAAQAAAgAAAAABAAAKAAAAAAEAAAQAAAAAAQAAAAAAAAABAAACAAAAAAEAAAoAAAAAAQAABAAAAAABAAAAAAAAAAEAAAIAAAAAAQAACgAAAAABAAAEAAAAAAEAAAAAAAAAAQAAAgAAAAABAAAEAAAAAAEAAAoAAAAAAQAABAAAAAABAAAAAAAAAAEAAAIAAAAAAQAACgAAAAABAAAEAAAAAAEAAAAAAAAAAQAAAgAAAAABAAAEAAAAAAEAAAoAAAAAAQAABAAAAAABAAAAAAAAAAEAAAIAAAAAAQAABgAAAAABAAACAAAAAAEAAAgAAAAAAgAAAgAAAAABAAAIAAAAAAIAAAIAAAAAAQAABAAAAAABAAAGAAAAAAEAAAIAAAAAAQAACgAAAAABAAAEAAAAAAEAAAAAAAAAAQAAAgAAAAABAAAKAAAAAAEAAAQAAAAAAQAAAAAAAAABAAACAAAAAAEAAAQAAAAAAQAACAAAAAACAAACAAAAAAMAAAQAAAAAAQAACgAAAAABAAAEAAAAAAEAAAAAAAAAAQAAAgAAAAABAAAIAAAAAAIAAAIAAAAAAQAACgAAAAABAAAEAAAAAAEAAAAAAAAAAQAAAgAAAAABAAAKAAAAAAEAAAQAAAAAAQAAAAAAAAABAAACAAAAAAEAAAoAAAAAAQAABAAAAAABAAAAAAAAAAEAAAIAAAAAAQAACgAAAAABAAAEAAAAAAEAAAAAAAAAAQAAAgAAAAABAAAKAAAAAAEAAAQAAAAAAQAAAAAAAAABAAACAAAAAAEAAAYAAAAAAQAAAgAAAAABAAAKAAAAAAEAAAQAAAAAAQAAAAAAAAABAAACAAAAAAEAAAoAAAAAAQAABAAAAAABAAAAAAAAAAEAAAIAAAAAAQAACgAAAAABAAAEAAAAAAEAAAAAAAAAAQAAAgAAAAABAAAKAAAAAAEAAAQAAAAAAQAAAAAAAAABAAACAAAAAAEAAAoAAAAAAQAABAAAAAABAAAAAAAAAAEAAAIAAAAAAQAACgAAAAABAAAEAAAAAAEAAAAAAAAAAQAAAgAAAAABAAAKAAAAAAEAAAQAAAAAAQAAAAAAAAABAAACAAAAAAEAAAoAAAAAAQAABAAAAAABAAAAAAAAAAEAAAIAAAAAAQAACgAAAAABAAAEAAAAAAEAAAAAAAAAAQAAAgAAAAABAAAKAAAAAAEAAAQAAAAAAQAAAAAAAAABAAACAAAAAAEAAAoAAAAAAQAABAAAAAABAAAAAAAAAAEAAAIAAAAAAQAABAAAAAABAAAGAAAAAAEAAAIAAAAAAQAABAAAAAABAAAGAAAAAAEAAAIAAAAAHHN0c2MAAAAAAAAAAQAAAAEAAADsAAAAAQAAA8RzdHN6AAAAAAAAAAAAAADsAAAEuAAAAJMAAAAqAAAAGQAAACwAAABnAAAANAAAABQAAAAzAAAAOwAAADUAAABUAAAAHwAAABIAAAA9AAAAOAAAACUAAAAaAAAAUQAAADwAAAAaAAAAGgAAAB0AAAAvAAAAIwAAAB8AAAAWAAAAEgAAAA8AAAAPAAAARwAAACYAAAAWAAAAIgAAACsAAAAnAAAAJAAAACIAAAArAAAAJwAAACQAAAAiAAAALQAAACQAAAAiAAAAHwAAABwAAAAVAAAAOgAAABcAAAAhAAAAIQAAACkAAAAkAAAAIwAAACAAAAA+AAAAIwAAACEAAAAfAAAAMwAAACQAAAAjAAAAIAAAACoAAAAkAAAAIQAAAB8AAABCAAAAJQAAABwAAAAnAAAAiQAAAC8AAAApAAAANgAAADsAAAAvAAAAPQAAAHcAAAAyAAAAVQAAADgAAAAwAAAAKwAAAFYAAAA6AAAALQAAADQAAAAoAAAAPAAAADYAAABBAAAALwAAACkAAABLAAAANgAAACcAAAAmAAAAQAAAACcAAAA/AAAAOQAAACUAAAAiAAAAQwAAADkAAAAiAAAAJAAAAEAAAAA5AAAAJAAAACIAAABQAAAAPQAAACUAAAAoAAAAqQAAAIoAAAAqAAAAKQAAACMAAAArAAAAIwAAACEAAAAgAAAANgAAADQAAAApAAAAJQAAACEAAAAzAAAAKAAAADEAAAAxAAAAJgAAADEAAAAqAAAAJAAAAG8AAACKAAAAawAAACgAAAArAAAAJQAAAC8AAAAnAAAAFgAAAA0AAAAVAAAAJQAAAJAAAAAdAAAAGAAAACMAAAAkAAAALAAAAEQAAABEAAAAOAAAAB4AAABLAAAAPwAAAB8AAABDAAAAOQAAABwAAAAXAAAAPgAAADYAAAAZAAAAFQAAAB0AAAArAAAAIAAAAB0AAAAWAAAADwAAAA0AAAANAAAAWQAAABcAAAARAAAAEgAAAD8AAAAWAAAAPQAAADYAAAAZAAAAFQAAAD4AAAA2AAAAGAAAABUAAAA+AAAANgAAABkAAAAVAAAAHQAAACsAAAAgAAAAHQAAABYAAAAPAAAADQAAAA0AAAAvAAAAEQAAAA8AAAANAAAAOAAAADIAAAASAAAAFgAAAEMAAAA6AAAAGgAAABkAAABDAAAAOQAAAB0AAAAVAAAAngAAAD4AAAAZAAAAHQAAAC4AAAAwAAAAJQAAAB0AAAAjAAAAIgAAABIAAAAeAAAAIwAAABMAAAAUc3RjbwAAAAAAAAABAAAAMAAAAGJ1ZHRhAAAAWm1ldGEAAAAAAAAAIWhkbHIAAAAAAAAAAG1kaXJhcHBsAAAAAAAAAAAAAAAALWlsc3QAAAAlqXRvbwAAAB1kYXRhAAAAAQAAAABMYXZmNTguMjAuMTAw\" type=\"video/mp4\" /></video>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import gym\n",
    "\n",
    "from gym import wrappers\n",
    "import io\n",
    "import base64\n",
    "from IPython.display import HTML\n",
    "\n",
    "env = gym.make('BreakoutDeterministic-v4') # frame skipping \n",
    "env = wrappers.Monitor(env, \"./gym-results\", force=True) # Cache game\n",
    "\n",
    "frame = env.reset() \n",
    "# env.render()\n",
    "\n",
    "is_done = False\n",
    "while not is_done: \n",
    "    frame, reward, is_done, _ = env.step(env.action_space.sample())\n",
    "#     env.render()\n",
    "env.close()\n",
    "    \n",
    "video = io.open('./gym-results/openaigym.video.%s.video000000.mp4' % env.file_infix, 'r+b').read()\n",
    "encoded = base64.b64encode(video)\n",
    "HTML(data='''\n",
    "    <video width=\"360\" height=\"auto\" alt=\"test\" controls><source src=\"data:video/mp4;base64,{0}\" type=\"video/mp4\" /></video>'''\n",
    ".format(encoded.decode('ascii')))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQcAAAD4CAYAAADhGCPfAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAL+klEQVR4nO3dX4id9Z3H8fcnk6RSNP6rxEGDujQ3sRTbDmF70XWhKcS9SITubpVKDQi5cIUupReBgBd6Ey39c6GwG9yFrDfWCksDpljNVnpTXVPqCrZoUtlibDRrtwglWFfy3Ys87o7Dd2ZSz5+ZJO8XhHme8/w4v69H5z3nHIecVBWStNCalR5A0upkHCS1jIOklnGQ1DIOklprV3qAxWTNx2vNzKUrPYZ0Xjv9/ptvV9VV3bVVG4c1M5dy0WW7VnoM6bx26u19v1nsmi8rJLWMg6SWcZDUMg6SWsZBUss4SGoZB0kt4yCpZRwktYyDpJZxkNQyDpJaxkFSyzhIahkHSS3jIKllHCS1RopDkiuSPJ3k6PD18iXWbkhyPMlDo+wpaTpGfeawBzhcVZuBw8P5Yu4HfjrifpKmZNQ47AQODMcHgFu7RUk+B2wEfjzifpKmZNQ4bKyqE8Pxm5wJwIckWQN8G/jmcneWZHeSI0mO1OlTI44maRTL/u3TSZ4Brm4u7Z1/UlWVpPtU3ruBQ1V1PMmSe1XVfmA/wMy6WT/hV1pBy8ahqrYtdi3JW0lmq+pEklngZLPs88AXktwNXAysT/KHqlrq/QlJK2zUz604CNwJ7Bu+/nDhgqr66gfHSXYBc4ZBWv1Gfc9hH/ClJEeBbcM5SeaSPDLqcJJWTqpW50v7mXWz5SdeSZN16u19P6+que6avyEpqWUcJLWMg6SWcZDUMg6SWsZBUss4SGoZB0kt4yCpZRwktYyDpJZxkNQyDpJaxkFSyzhIahkHSS3jIKllHCS1jIOklnGQ1DIOklrGQVLLOEhqGQdJLeMgqWUcJLWMg6SWcZDUMg6SWsZBUss4SGoZB0kt4yCpZRwktUaKQ5Irkjyd5Ojw9fJmzU1Jfpbk5SQvJfnKKHtKmo5RnznsAQ5X1Wbg8HC+0Cnga1V1I7Ad+F6Sy0bcV9KEjRqHncCB4fgAcOvCBVX1alUdHY5/C5wErhpxX0kTNmocNlbVieH4TWDjUouTbAXWA78ecV9JE7Z2uQVJngGubi7tnX9SVZWklrifWeBR4M6qOr3Imt3AboCs2bDcaJImaNk4VNW2xa4leSvJbFWdGL75Ty6ybgPwJLC3qp5bYq/9wH6AmXWzi4ZG0uSN+rLiIHDncHwn8MOFC5KsB/4V+JeqemLE/SRNyahx2Ad8KclRYNtwTpK5JI8Ma/4W+AtgV5IXhz83jbivpAlL1ep89j6zbrYuumzXSo8hnddOvb3v51U1113zNyQltYyDpJZxkNQyDpJaxkFSyzhIahkHSS3jIKllHCS1jIOklnGQ1DIOklrGQVLLOEhqGQdJLeMgqWUcJLWMg6SWcZDUMg6SWsZBUss4SGoZB0kt4yCpZRwktYyDpJZxkNQyDpJaxkFSyzhIahkHSS3jIKllHCS1jIOklnGQ1BpLHJJsT/JKkmNJ9jTXP5bk+8P155NcP459JU3OyHFIMgM8DNwCbAFuT7JlwbK7gN9X1SeB7wIPjLqvpMkaxzOHrcCxqnqtqt4DHgN2LlizEzgwHD8BfDFJxrC3pAkZRxyuAV6fd358uK1dU1XvA+8AVy68oyS7kxxJcqROnxrDaJI+qlX1hmRV7a+quaqay5qPr/Q40gVtHHF4A9g07/za4bZ2TZK1wKXA78awt6QJGUccXgA2J7khyXrgNuDggjUHgTuH478G/q2qagx7S5qQtaPeQVW9n+Qe4ClgBvjnqno5yX3Akao6CPwT8GiSY8B/cyYgklaxkeMAUFWHgEMLbrt33vG7wN+MYy9J07Gq3pCUtHoYB0kt4yCpZRwktYyDpJZxkNQyDpJaxkFSyzhIahkHSS3jIKllHCS1jIOklnGQ1DIOklrGQVLLOEhqGQdJLeMgqWUcJLWMg6SWcZDUMg6SWsZBUss4SGoZB0kt4yCpZRwktYyDpJZxkNQyDpJaxkFSyzhIahkHSa2xxCHJ9iSvJDmWZE9z/RtJfpnkpSSHk1w3jn0lTc7IcUgyAzwM3AJsAW5PsmXBsl8Ac1X1aeAJ4MFR95U0WeN45rAVOFZVr1XVe8BjwM75C6rqJ1V1ajh9Drh2DPtKmqBxxOEa4PV558eH2xZzF/CjMewraYLWTnOzJHcAc8DNi1zfDewGyJoNU5xM0kLjiMMbwKZ559cOt31Ikm3AXuDmqvpjd0dVtR/YDzCzbrbGMJukj2gcLyteADYnuSHJeuA24OD8BUk+A/wjsKOqTo5hT0kTNnIcqup94B7gKeBXwONV9XKS+5LsGJZ9C7gY+EGSF5McXOTuJK0SY3nPoaoOAYcW3HbvvONt49hH0vT4G5KSWsZBUss4SGoZB0kt4yCpZRwktYyDpJZxkNQyDpJaxkFSyzhIahkHSS3jIKllHCS1jIOklnGQ1DIOklrGQVLLOEhqGQdJLeMgqWUcJLWMg6SWcZDUMg6SWsZBUss4SGoZB0kt4yCpZRwktYyDpJZxkNQyDpJaxkFSyzhIao0lDkm2J3klybEke5ZY9+UklWRuHPtKmpyR45BkBngYuAXYAtyeZEuz7hLg68Dzo+4pafLG8cxhK3Csql6rqveAx4Cdzbr7gQeAd8ewp6QJG0ccrgFen3d+fLjt/yT5LLCpqp5c6o6S7E5yJMmROn1qDKNJ+qjWTnqDJGuA7wC7lltbVfuB/QAz62ZrspNJWso4njm8AWyad37tcNsHLgE+BTyb5D+BPwcO+qaktLqNIw4vAJuT3JBkPXAbcPCDi1X1TlV9oqqur6rrgeeAHVV1ZAx7S5qQkeNQVe8D9wBPAb8CHq+ql5Pcl2THqPcvaWWM5T2HqjoEHFpw272LrP3LcewpabL8DUlJLeMgqWUcJLWMg6SWcZDUMg6SWsZBUss4SGoZB0kt4yCpZRwktYyDpJZxkNQyDpJaxkFSyzhIaqVqdf49rkn+C/jNBO76E8DbE7jfSTmX5j2XZoVza95JzXpdVV3VXVi1cZiUJEeq6pz5y23PpXnPpVnh3Jp3JWb1ZYWklnGQ1LoQ47B/pQf4E51L855Ls8K5Ne/UZ73g3nOQdHYuxGcOks6CcZDUOu/jkOSKJE8nOTp8vXyJtRuSHE/y0DRnXDDDsvMmuSnJz5K8nOSlJF+Z8ozbk7yS5FiSPc31jyX5/nD9+STXT3O+BbMsN+s3kvxyeBwPJ7luJeacN8+S885b9+UkNcnPnD3v4wDsAQ5X1Wbg8HC+mPuBn05lqsWdzbyngK9V1Y3AduB7SS6bxnBJZoCHgVuALcDtSbYsWHYX8Puq+iTwXeCBacy20FnO+gtgrqo+DTwBPDjdKf/fWc5LkkuArwPPT3KeCyEOO4EDw/EB4NZuUZLPARuBH09prsUsO29VvVpVR4fj3wIngfa33CZgK3Csql6rqveAxzgz83zz/xmeAL6YJFOab75lZ62qn1TVqeH0Oc58SvxKOZvHFs78EHsAeHeSw1wIcdhYVSeG4zc5E4APSbIG+DbwzWkOtohl550vyVZgPfDrSQ82uAZ4fd758eG2ds3wQcvvAFdOZbpF5hh0s853F/CjiU60tGXnTfJZYFNVPTnpYcbyQborLckzwNXNpb3zT6qqknT/7/Zu4FBVHZ/GD7gxzPvB/cwCjwJ3VtXp8U55YUlyBzAH3LzSsyxm+CH2HWDXNPY7L+JQVdsWu5bkrSSzVXVi+GY62Sz7PPCFJHcDFwPrk/yhqpZ6f2Il5yXJBuBJYG9VPTeJORfxBrBp3vm1w23dmuNJ1gKXAr+bznjtHB/oZiXJNs6E+eaq+uOUZussN+8lwKeAZ4cfYlcDB5PsqKojY5+mqs7rP8C3gD3D8R7gwWXW7wIeWs3zcuZlxGHg71dgvrXAa8ANwxz/Ady4YM3fAf8wHN8GPL5Cj+XZzPoZzrwk27xS/87/lHkXrH+WM2+mTmaelX5ApvCAXzl8Ix0FngGuGG6fAx5p1q90HJadF7gD+B/gxXl/bprijH8FvDp8U+0dbrsP2DEcXwT8ADgG/DvwZyv4eC436zPAW/Mex4Mr/N/rkvMuWDvROPjr05JaF8L/rZD0ERgHSS3jIKllHCS1jIOklnGQ1DIOklr/C6ycA7eW4fkmAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[50]]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQcAAAD4CAYAAADhGCPfAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAL70lEQVR4nO3dX6jf9X3H8edLsyitpjVWYlBRR3MTS7HtIV0vOgemoLtIhG6rMmkEIRdO6Ci9CAS80Bu19M+FwhbcIPPG2sBowBRrskpvqjOlTrBFk8qKsdGsnciK+I++d5Gv28/D+5yT+vt3kjwfEM73+/t++H3e/vQ8z+/385BfqgpJWuyceQ8gaXUyDpJaxkFSyzhIahkHSa018x5gKWtzXp3PR+c9hnRG+x9e/21VXdJdW7VxOJ+P8vlcP+8xpDPawdr366Wu+bJCUss4SGoZB0kt4yCpZRwktYyDpJZxkNQyDpJaxkFSyzhIahkHSS3jIKllHCS1jIOklnGQ1DIOklrGQVJrrDgkWZ/kiSRHhq8XLbN2XZJjSR4YZ09JszHuM4ddwKGq2gQcGs6Xcg/wkzH3kzQj48ZhO7B3ON4L3NQtSvI5YAPwozH3kzQj48ZhQ1UdH45f5WQAPiDJOcC3gG+sdGdJdiY5nOTwu7w95miSxrHi3z6d5CBwaXNp9+hJVVWS7lN57wAOVNWxJMvuVVV7gD0A67LeT/iV5mjFOFTV1qWuJXktycaqOp5kI3CiWfYF4ItJ7gAuANYm+X1VLff+hKQ5G/dzK/YDO4B7h68/WLygqv72/eMktwELhkFa/cZ9z+Fe4EtJjgBbh3OSLCR5aNzhJM1PqlbnS/t1WV9+4pU0XQdr38+qaqG75m9ISmoZB0kt4yCpZRwktYyDpJZxkNQyDpJaxkFSyzhIahkHSS3jIKllHCS1jIOklnGQ1DIOklrGQVLLOEhqGQdJLeMgqWUcJLWMg6SWcZDUMg6SWsZBUss4SGoZB0kt4yCpZRwktYyDpJZxkNQyDpJaxkFSyzhIahkHSa2x4pBkfZInkhwZvl7UrLk2yU+TPJ/kuSRfGWdPSbMx7jOHXcChqtoEHBrOF3sT+GpVXQPcAHw3ycfH3FfSlI0bh+3A3uF4L3DT4gVV9WJVHRmOfwOcAC4Zc19JUzZuHDZU1fHh+FVgw3KLk2wB1gK/GnNfSVO2ZqUFSQ4ClzaXdo+eVFUlqWXuZyPwMLCjqv6wxJqdwE6A8/nISqNJmqIV41BVW5e6luS1JBur6vjwzX9iiXXrgMeA3VX11DJ77QH2AKzL+iVDI2n6xn1ZsR/YMRzvAH6weEGStcC/Av9SVfvG3E/SjIwbh3uBLyU5AmwdzkmykOShYc3fAH8O3Jbk2eHPtWPuK2nKUrU6n72vy/r6fK6f9xjSGe1g7ftZVS101/wNSUkt4yCpZRwktYyDpJZxkNQyDpJaxkFSyzhIahkHSS3jIKllHCS1jIOklnGQ1DIOklrGQVLLOEhqGQdJLeMgqWUcJLWMg6SWcZDUMg6SWsZBUss4SGoZB0kt4yCpZRwktYyDpJZxkNQyDpJaxkFSyzhIahkHSS3jIKllHCS1JhKHJDckeSHJ0SS7muvnJfnecP3pJFdNYl9J0zN2HJKcCzwI3AhsBm5JsnnRstuB16vqk8B3gPvG3VfSdE3imcMW4GhVvVRV7wCPANsXrdkO7B2O9wHXJ8kE9pY0JZOIw2XAyyPnx4bb2jVV9R7wBnDx4jtKsjPJ4SSH3+XtCYwm6cNaVW9IVtWeqlqoqoU/4bx5jyOd1SYRh1eAK0bOLx9ua9ckWQN8DPjdBPaWNCWTiMMzwKYkVydZC9wM7F+0Zj+wYzj+K+DfqqomsLekKVkz7h1U1XtJ7gQeB84F/rmqnk9yN3C4qvYD/wQ8nOQo8N+cDIikVWzsOABU1QHgwKLb7ho5fgv460nsJWk2VtUbkpJWD+MgqWUcJLWMg6SWcZDUMg6SWsZBUss4SGoZB0kt4yCpZRwktYyDpJZxkNQyDpJaxkFSyzhIahkHSS3jIKllHCS1jIOklnGQ1DIOklrGQVLLOEhqGQdJLeMgqWUcJLWMg6SWcZDUMg6SWsZBUss4SGoZB0kt4yCpNZE4JLkhyQtJjibZ1Vz/epJfJHkuyaEkV05iX0nTM3YckpwLPAjcCGwGbkmyedGynwMLVfVpYB9w/7j7SpquSTxz2AIcraqXquod4BFg++iCqvpxVb05nD4FXD6BfSVN0STicBnw8sj5seG2pdwO/HAC+0qaojWz3CzJrcACcN0S13cCOwHO5yMznEzSYpOIwyvAFSPnlw+3fUCSrcBu4Lqqeru7o6raA+wBWJf1NYHZJH1Ik3hZ8QywKcnVSdYCNwP7Rxck+Qzwj8C2qjoxgT0lTdnYcaiq94A7gceBXwKPVtXzSe5Osm1Y9k3gAuD7SZ5Nsn+Ju5O0SkzkPYeqOgAcWHTbXSPHWyexj6TZ8TckJbWMg6SWcZDUMg6SWsZBUss4SGoZB0kt4yCpZRwktYyDpJZxkNQyDpJaxkFSyzhIahkHSS3jIKllHCS1jIOklnGQ1DIOklrGQVLLOEhqGQdJLeMgqWUcJLWMg6SWcZDUMg6SWsZBUss4SGoZB0kt4yCpZRwktYyDpJZxkNSaSByS3JDkhSRHk+xaZt2Xk1SShUnsK2l6xo5DknOBB4Ebgc3ALUk2N+suBL4GPD3unpKmbxLPHLYAR6vqpap6B3gE2N6suwe4D3hrAntKmrJJxOEy4OWR82PDbf8nyWeBK6rqseXuKMnOJIeTHH6XtycwmqQPa820N0hyDvBt4LaV1lbVHmAPwLqsr+lOJmk5k3jm8Apwxcj55cNt77sQ+BTwZJL/BP4M2O+bktLqNok4PANsSnJ1krXAzcD+9y9W1RtV9YmquqqqrgKeArZV1eEJ7C1pSsaOQ1W9B9wJPA78Eni0qp5PcneSbePev6T5mMh7DlV1ADiw6La7llj7F5PYU9J0+RuSklrGQVLLOEhqGQdJLeMgqWUcJLWMg6SWcZDUMg6SWsZBUss4SGoZB0kt4yCpZRwktYyDpJZxkNRK1er8e1yT/Bfw6ync9SeA307hfqfldJr3dJoVTq95pzXrlVV1SXdh1cZhWpIcrqrT5i+3PZ3mPZ1mhdNr3nnM6ssKSS3jIKl1NsZhz7wH+COdTvOeTrPC6TXvzGc9695zkHRqzsZnDpJOgXGQ1Drj45BkfZInkhwZvl60zNp1SY4leWCWMy6aYcV5k1yb5KdJnk/yXJKvzHjGG5K8kORokl3N9fOSfG+4/nSSq2Y536JZVpr160l+MTyOh5JcOY85R+ZZdt6RdV9OUtP8zNkzPg7ALuBQVW0CDg3nS7kH+MlMplraqcz7JvDVqroGuAH4bpKPz2K4JOcCDwI3ApuBW5JsXrTsduD1qvok8B3gvlnMttgpzvpzYKGqPg3sA+6f7ZT/7xTnJcmFwNeAp6c5z9kQh+3A3uF4L3BTtyjJ54ANwI9mNNdSVpy3ql6sqiPD8W+AE0D7W25TsAU4WlUvVdU7wCOcnHnU6D/DPuD6JJnRfKNWnLWqflxVbw6nT3HyU+Ln5VQeWzj5Q+w+4K1pDnM2xGFDVR0fjl/lZAA+IMk5wLeAb8xysCWsOO+oJFuAtcCvpj3Y4DLg5ZHzY8Nt7Zrhg5bfAC6eyXRLzDHoZh11O/DDqU60vBXnTfJZ4Iqqemzaw0zkg3TnLclB4NLm0u7Rk6qqJN3/u70DOFBVx2bxA24C875/PxuBh4EdVfWHyU55dklyK7AAXDfvWZYy/BD7NnDbLPY7I+JQVVuXupbktSQbq+r48M10oln2BeCLSe4ALgDWJvl9VS33/sQ85yXJOuAxYHdVPTWNOZfwCnDFyPnlw23dmmNJ1gAfA343m/HaOd7XzUqSrZwM83VV9faMZuusNO+FwKeAJ4cfYpcC+5Nsq6rDE5+mqs7oP8A3gV3D8S7g/hXW3wY8sJrn5eTLiEPA389hvjXAS8DVwxz/AVyzaM3fAf8wHN8MPDqnx/JUZv0MJ1+SbZrXv/M/Zt5F65/k5Jup05ln3g/IDB7wi4dvpCPAQWD9cPsC8FCzft5xWHFe4FbgXeDZkT/XznDGvwReHL6pdg+33Q1sG47PB74PHAX+HfjTOT6eK816EHht5HHcP+f/Xpedd9HaqcbBX5+W1Dob/m+FpA/BOEhqGQdJLeMgqWUcJLWMg6SWcZDU+l8Pt/+r0QIe9gAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "array([[50]], dtype=uint8)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt \n",
    "%matplotlib inline\n",
    "\n",
    "pixel_example = np.array([ [ [10, 20, 120] ] ]) #, [20, 40, 30] ] ])\n",
    "plt.imshow(pixel_example)\n",
    "plt.show()\n",
    "\n",
    "def convert_rgb_to_grayscale(image): \n",
    "    grayscale = np.mean(image, axis=2).astype(np.uint8)\n",
    "    print(grayscale)\n",
    "    plt.imshow(grayscale)\n",
    "    plt.show()\n",
    "    return np.mean(image, axis=2).astype(np.uint8)\n",
    "\n",
    "# assert True == False\n",
    "\n",
    "convert_rgb_to_grayscale(pixel_example)\n",
    "\n",
    "# def downsample(img): "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Preprocess \n",
    "def to_grayscale(img):\n",
    "    return np.mean(img, axis=2).astype(np.uint8)\n",
    "\n",
    "def downsample(img):\n",
    "    return img[::2, ::2]\n",
    "\n",
    "def preprocess(img):\n",
    "    return to_grayscale(downsample(img))\n",
    "\n",
    "# Normalize rewards \n",
    "def transform_reward(reward):\n",
    "        return np.sign(reward)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Ring Buffer \n",
    "class RingBuf:\n",
    "    def __init__(self, size):\n",
    "        # Pro-tip: when implementing a ring buffer, always allocate one extra element,\n",
    "        # this way, self.start == self.end always means the buffer is EMPTY, whereas\n",
    "        # if you allocate exactly the right number of elements, it could also mean\n",
    "        # the buffer is full. This greatly simplifies the rest of the code.\n",
    "        self.data = [None] * (size + 1)\n",
    "        self.start = 0\n",
    "        self.end = 0\n",
    "        \n",
    "    def append(self, element):\n",
    "        self.data[self.end] = element\n",
    "        self.end = (self.end + 1) % len(self.data)\n",
    "        # end == start and yet we just added one element. This means the buffer has one\n",
    "        # too many element. Remove the first element by incrementing start.\n",
    "        if self.end == self.start:\n",
    "            self.start = (self.start + 1) % len(self.data)\n",
    "        \n",
    "    def __getitem__(self, idx):\n",
    "        return self.data[(self.start + idx) % len(self.data)]\n",
    "    \n",
    "    def __len__(self):\n",
    "        if self.end < self.start:\n",
    "            return self.end + len(self.data) - self.start\n",
    "        else:\n",
    "            return self.end - self.start\n",
    "        \n",
    "    def __iter__(self):\n",
    "        for i in range(len(self)):\n",
    "            yield self[i]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Fit \n",
    "def fit_batch(model, gamma, start_states, actions, rewards, next_states, is_terminal):\n",
    "    \"\"\"Do one deep Q learning iteration.\n",
    "    \n",
    "    Params:\n",
    "    - model: The DQN\n",
    "    - gamma: Discount factor (should be 0.99)\n",
    "    - start_states: numpy array of starting states\n",
    "    - actions: numpy array of one-hot encoded actions corresponding to the start states\n",
    "    - rewards: numpy array of rewards corresponding to the start states and actions\n",
    "    - next_states: numpy array of the resulting states corresponding to the start states and actions\n",
    "    - is_terminal: numpy boolean array of whether the resulting state is terminal\n",
    "    \n",
    "    \"\"\"\n",
    "    # First, predict the Q values of the next states. Note how we are passing ones as the mask.\n",
    "    next_Q_values = model.predict([next_states, np.ones(actions.shape)])\n",
    "    # The Q values of the terminal states is 0 by definition, so override them\n",
    "    next_Q_values[is_terminal] = 0\n",
    "    # The Q values of each start state is the reward + gamma * the max next state Q value\n",
    "    Q_values = rewards + gamma * np.max(next_Q_values, axis=1)\n",
    "    # Fit the keras model. Note how we are passing the actions as the mask and multiplying\n",
    "    # the targets by the actions.\n",
    "    model.fit(\n",
    "        [start_states, actions], actions * Q_values[:, None],\n",
    "        nb_epoch=1, batch_size=len(start_states), verbose=0\n",
    "    )\n",
    "\n",
    "# Model \n",
    "def atari_model(n_actions):\n",
    "    # We assume a theano backend here, so the \"channels\" are first.\n",
    "    ATARI_SHAPE = (4, 105, 80)\n",
    "\n",
    "    # With the functional API we need to define the inputs.\n",
    "    frames_input = keras.layers.Input(ATARI_SHAPE, name='frames')\n",
    "    actions_input = keras.layers.Input((n_actions,), name='mask')\n",
    "\n",
    "    # Assuming that the input frames are still encoded from 0 to 255. Transforming to [0, 1].\n",
    "    normalized = keras.layers.Lambda(lambda x: x / 255.0)(frames_input)\n",
    "    \n",
    "    # \"The first hidden layer convolves 16 8×8 filters with stride 4 with the input image and applies a rectifier nonlinearity.\"\n",
    "    conv_1 = keras.layers.convolutional.Convolution2D(\n",
    "        16, 8, 8, subsample=(4, 4), activation='relu'\n",
    "    )(normalized)\n",
    "    # \"The second hidden layer convolves 32 4×4 filters with stride 2, again followed by a rectifier nonlinearity.\"\n",
    "    conv_2 = keras.layers.convolutional.Convolution2D(\n",
    "        32, 4, 4, subsample=(2, 2), activation='relu'\n",
    "    )(conv_1)\n",
    "    # Flattening the second convolutional layer.\n",
    "    conv_flattened = keras.layers.core.Flatten()(conv_2)\n",
    "    # \"The final hidden layer is fully-connected and consists of 256 rectifier units.\"\n",
    "    hidden = keras.layers.Dense(256, activation='relu')(conv_flattened)\n",
    "    # \"The output layer is a fully-connected linear layer with a single output for each valid action.\"\n",
    "    output = keras.layers.Dense(n_actions)(hidden)\n",
    "    # Finally, we multiply the output by the mask!\n",
    "    filtered_output = keras.layers.merge([output, actions_input], mode='mul')\n",
    "\n",
    "    self.model = keras.models.Model(input=[frames_input, actions_input], output=filtered_output)\n",
    "    optimizer = optimizer=keras.optimizers.RMSprop(lr=0.00025, rho=0.95, epsilon=0.01)\n",
    "    self.model.compile(optimizer, loss='mse')\n",
    "            \n",
    "def q_iteration(env, model, state, iteration, memory):\n",
    "    # Choose epsilon based on the iteration\n",
    "    epsilon = get_epsilon_for_iteration(iteration)\n",
    "\n",
    "    # Choose the action \n",
    "    if random.random() < epsilon:\n",
    "        action = env.action_space.sample()\n",
    "    else:\n",
    "        action = choose_best_action(model, state)\n",
    "\n",
    "    # Play one game iteration (note: according to the next paper, you should actually play 4 times here)\n",
    "    new_frame, reward, is_done, _ = env.step(action)\n",
    "    memory.add(state, action, new_frame, reward, is_done)\n",
    "\n",
    "    # Sample and fit\n",
    "    batch = memory.sample_batch(32)\n",
    "    fit_batch(model, batch)\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}