cnn_cifar10.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "cnn_cifar10.ipynb",
      "version": "0.3.2",
      "provenance": []
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.6.8"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "metadata": {
        "colab_type": "text",
        "id": "EItZAeAcbfho"
      },
      "cell_type": "markdown",
      "source": [
        "# Convolutional Neural Networks"
      ]
    },
    {
      "metadata": {
        "colab_type": "code",
        "id": "I04c9PFJbfhp",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "import tensorflow as tf\n",
        "\n",
        "%matplotlib inline"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "colab_type": "code",
        "id": "OHnawNjVbfht",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# option 1: MNIST\n",
        "(X_train, Y_train), (X_test, Y_test) = tf.keras.datasets.mnist.load_data()\n",
        "X_train = np.reshape(X_train, [-1, 28, 28, 1])/255.\n",
        "X_test = np.reshape(X_test, [-1, 28, 28, 1])/255."
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "colab_type": "code",
        "id": "Uji6BsH2bfhw",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# option 2: CIFAR10\n",
        "proportion_to_use = 0.3\n",
        "(X_train, Y_train), (X_test, Y_test) = tf.keras.datasets.cifar10.load_data()\n",
        "Y_train = Y_train[0:int(np.shape(Y_train)[0]*proportion_to_use),0]\n",
        "Y_test = Y_test[0:int(np.shape(Y_test)[0]*proportion_to_use),0]\n",
        "X_train = X_train[0:int(np.shape(X_train)[0]*proportion_to_use),:]/255.\n",
        "X_test = X_test[0:int(np.shape(X_test)[0]*proportion_to_use),:]/255."
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "colab_type": "code",
        "id": "yYrUoyzsbfhz",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def random_mini_batches(X, Y, minibatch_size = 64, seed = 0):\n",
        "\n",
        "    m = X.shape[0]\n",
        "    minibatches = []\n",
        "    np.random.seed(seed)\n",
        "\n",
        "    perm = list(np.random.permutation(m))\n",
        "    permX = X[perm,:,:,:]\n",
        "    permY = Y[perm]\n",
        "\n",
        "    n_full = int(np.floor(m/minibatch_size)) \n",
        "    for k in range(0, n_full):\n",
        "        miniX = permX[k * minibatch_size : k * minibatch_size + minibatch_size,:,:,:]\n",
        "        miniY = permY[k * minibatch_size : k * minibatch_size + minibatch_size]\n",
        "        minibatches.append((miniX, miniY))\n",
        "\n",
        "    if m % minibatch_size != 0:\n",
        "        miniX = permX[n_full * minibatch_size : m,:,:,:]\n",
        "        miniY = permY[n_full * minibatch_size : m]\n",
        "        minibatches.append((miniX, miniY))\n",
        "\n",
        "    return minibatches\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "colab_type": "code",
        "id": "_YBASTbpbfh2",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# create placeholders\n",
        "\n",
        "def create_placeholders(n_H0, n_W0, n_C0):\n",
        "    X = tf.placeholder(tf.float32, shape=(None, n_H0, n_W0, n_C0))\n",
        "    Y = tf.placeholder(tf.float32, shape=(None))\n",
        "    \n",
        "    return X, Y"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "colab_type": "code",
        "id": "5j6g5oNRbfh4",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# forward propagation\n",
        "\n",
        "def forward_propagation(X):\n",
        "    \n",
        "    A1 = tf.layers.conv2d(X, filters=128, kernel_size=[3,3], strides = (1,1), \n",
        "                          padding = 'SAME', activation=tf.nn.relu)\n",
        "    P1 = tf.layers.max_pooling2d(A1, pool_size=[4, 4], strides=4, padding='SAME')\n",
        "    D1 = tf.layers.dropout(P1, rate=0.25)\n",
        "    D1 = tf.layers.batch_normalization(D1)\n",
        "\n",
        "    \n",
        "    A2 = tf.layers.conv2d(D1, filters=64, kernel_size=[3,3], strides=(1,1),\n",
        "                         padding = 'SAME', activation=tf.nn.relu)\n",
        "    P2 = tf.layers.max_pooling2d(A2, pool_size=[4, 4], strides=4, padding='SAME')\n",
        "    D2 = tf.layers.dropout(P2, rate=0.25)\n",
        "    D2 = tf.layers.batch_normalization(D2)\n",
        "    \n",
        "    flat_dim = np.product(D2.get_shape()[1:])\n",
        "    \n",
        "    D2_flat = tf.reshape(D2,shape=[-1,flat_dim])\n",
        "    FC1 = tf.layers.dense(D2_flat, 2048, activation=tf.nn.relu)\n",
        "    D3 = tf.layers.dropout(FC1, rate=0.75)\n",
        "    D3 = tf.layers.batch_normalization(D3)\n",
        "    \n",
        "    Z3 = tf.layers.dense(D2_flat, 10)\n",
        "\n",
        "    return Z3"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "colab_type": "code",
        "id": "UMrnmm0ybfh7",
        "outputId": "f4fd94c7-cd92-477e-f9c8-b7a3cad013ee",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 887
        }
      },
      "cell_type": "code",
      "source": [
        "learning_rate = 0.009\n",
        "num_epochs = 60\n",
        "minibatch_size = int(X_train.shape[0]/50)\n",
        "    \n",
        "(m, n_H0, n_W0, n_C0) = X_train.shape                                        \n",
        "costs = []         \n",
        "seed = 0\n",
        "\n",
        "X, Y = create_placeholders(32, 32, 3)\n",
        "Z3 = forward_propagation(X)\n",
        "\n",
        "cost = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(logits = Z3, labels=tf.cast(Y, dtype=tf.int32)))\n",
        "optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)\n",
        "\n",
        "init = tf.global_variables_initializer()\n",
        " \n",
        "with tf.Session() as sess:\n",
        "    sess.run(init)\n",
        "        \n",
        "    for epoch in range(num_epochs):\n",
        "\n",
        "        minibatch_cost = 0.\n",
        "        num_minibatches = int(m / minibatch_size) \n",
        "        seed = seed + 1\n",
        "        minibatches = random_mini_batches(X_train, Y_train, minibatch_size, seed)\n",
        "\n",
        "        for minibatch in minibatches:\n",
        "\n",
        "            (minibatch_X, minibatch_Y) = minibatch\n",
        "            _ , temp_cost = sess.run([optimizer, cost], feed_dict={X: minibatch_X, Y: minibatch_Y})\n",
        "\n",
        "            minibatch_cost += temp_cost / num_minibatches\n",
        "\n",
        "        costs.append(minibatch_cost)\n",
        "\n",
        "        if epoch == 0 or (epoch+1)%5 == 0:\n",
        "            print(\"Cost after epoch %i: %f\" % (epoch+1, minibatch_cost))\n",
        " \n",
        "    correct_prediction = tf.equal(tf.argmax(Z3, 1), tf.cast(Y,\"int64\"))\n",
        "    accuracy = tf.reduce_mean(tf.cast(correct_prediction, \"float\"))\n",
        "    print(\"Training Accuracy:\", accuracy.eval({X: X_train, Y: Y_train}))\n",
        "    print(\"Testing Accuracy:\", accuracy.eval({X: X_test, Y: Y_test}))\n",
        "    \n",
        "    plt.plot(np.squeeze(costs))\n",
        "    plt.ylabel('cost')\n",
        "    plt.xlabel('iterations')\n",
        "    plt.title(\"Learning rate =\" + str(learning_rate))\n",
        "    plt.show()"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From <ipython-input-6-5e8e41616052>:5: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use keras.layers.conv2d instead.\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Colocations handled automatically by placer.\n",
            "WARNING:tensorflow:From <ipython-input-6-5e8e41616052>:6: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use keras.layers.max_pooling2d instead.\n",
            "WARNING:tensorflow:From <ipython-input-6-5e8e41616052>:7: dropout (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use keras.layers.dropout instead.\n",
            "WARNING:tensorflow:From <ipython-input-6-5e8e41616052>:8: batch_normalization (from tensorflow.python.layers.normalization) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use keras.layers.batch_normalization instead.\n",
            "WARNING:tensorflow:From <ipython-input-6-5e8e41616052>:20: dense (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use keras.layers.dense instead.\n",
            "Cost after epoch 1: 2.147643\n",
            "Cost after epoch 5: 1.398369\n",
            "Cost after epoch 10: 1.214412\n",
            "Cost after epoch 15: 1.083297\n",
            "Cost after epoch 20: 0.998739\n",
            "Cost after epoch 25: 0.917126\n",
            "Cost after epoch 30: 0.873676\n",
            "Cost after epoch 35: 0.840697\n",
            "Cost after epoch 40: 0.788130\n",
            "Cost after epoch 45: 0.772367\n",
            "Cost after epoch 50: 0.726244\n",
            "Cost after epoch 55: 0.728047\n",
            "Cost after epoch 60: 0.686276\n",
            "Training Accuracy: 0.7704667\n",
            "Testing Accuracy: 0.602\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEWCAYAAACJ0YulAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3Xl8XGXd///XJ2ubrWmatE3Tpum+\n0J0WqFAWi1oQRVG5RRRRsLKoxeWnqF/F+1bum1vcUASsCAVFQCkCIqveQIFCIV3pTlu6pFvSJWnT\ntM32+f1xTuNQkjRtM5lM5v18PObRzJxrZj5XO513znWucx1zd0RERACSYl2AiIh0HgoFERFpolAQ\nEZEmCgUREWmiUBARkSYKBRERaaJQkC7JzJ42s8/Hug6ReKNQkHZlZhvN7PxY1+HuF7j7fbGuA8DM\nXjSzqzvgfdLN7B4z22dmO8zsG8do//Ww3b7weekR20rM7AUzqzGz1ZH/puH7/NLMtpnZXjO7w8xS\no9k36TgKBYk7ZpYS6xqO6Ey1AD8ChgEDgfOAb5vZjOYamtmHgBuB6WH7wcB/RjR5EFgM9AK+Dzxi\nZgXhthuBycAYYDgwCfh/7dwXiRV31023drsBG4HzW9h2EbAEqATmA+Mitt0IrAf2AyuBj0dsuxJ4\nFfglsBv4SfjYK8DPgL3AO8AFEc95Ebg64vmttR0EzAvf+5/Ab4E/tdCHc4Ey4DvADuCPQE/gSaAi\nfP0ngf5h+5uBBuAQUA3cHj4+Enge2AOsAS5th7/7bcAHI+7/GHiohbZ/Bv474v50YEf483DgMJAd\nsf1l4Jrw51LgUxHbPgNsifVnT7f2uWlPQTqEmU0E7gG+TPDb5++AJyKGLNYD04AeBL+x/snMCiNe\n4nRgA9CH4Iv2yGNrgHzgp8AfzMxaKKG1tn8G3gjr+hHwuWN0py+QR/Ab9kyCPe57w/vFwEHgdgB3\n/z7BF+pX3D3L3b9iZpkEgfBnoDfwaeAOMxvd3JuFwzOVLdyWhW16AoXA0oinLgVOaaEPpzTTto+Z\n9Qq3bXD3/a28lh31c38z69HCe0kcUShIR5kJ/M7dF7h7gwfj/YeBMwDc/a/uvs3dG939YeBt4LSI\n529z99+4e727Hwwf2+Tuv3f3BuA+gi/FPi28f7NtzawYmAL80N1r3f0V4Ilj9KURuMndD7v7QXff\n7e5z3b0m/CK9GTinledfBGx093vD/iwG5gKfaq6xu1/n7rkt3MaFzbLCP6sinloFZLdQQ1YzbQnb\nH73t6Nd6BphlZgVm1hf4Wvh4Ros9lrjRmcZDpWsbCHzezL4a8Vga0A/AzK4AvgGUhNuyCH6rP2JL\nM6+548gP7l4T/uKf1Uy71trmA3vcveao9xrQSl8q3P3QkTtmlkEwtDWDYCgJINvMksMQOtpA4HQz\nq4x4LIVgKOpEVYd/5hAMVR35eX/zzakOtxPRlrD90duOfq2bgVyCocDDwO+BicDOE6xdOhHtKUhH\n2QLcfNRvuRnu/qCZDST4YvkK0Mvdc4HlvHuIIlrL+W4H8sIv9iNaC4TmavkmMAI43d1zgLPDx62F\n9luAl476u8hy92ubezMzu8vMqlu4rQBw971hX8ZHPHU8sKKFPqxopu1Od98dbhtsZtlHbT/yXgfd\n/SvuXuTugwmO8yx098YW3kviiEJBoiHVzLpF3FIIvvSvMbPTLZBpZh8Ov3gyCb44KwDM7AsEM1ui\nzt03ERw4/ZGZpZnZVOAjx/ky2QTHESrNLA+46ajtOwlm9xzxJDDczD5nZqnhbYqZjWqhxmvC0Gju\nFjnOfz/w/8ysp5mNBL4EzGmh5vuBq8xstJnlEswemhO+31qCvYCbwn+/jwPjCIa4MLMiM+sX/jue\nAfygmT5LnFIoSDQ8RfAleeT2I3cvJfiSup1ghs46gllBuPtK4OfAawRfoGMJZht1lMuBqfx7ZtPD\nBMMibfUroDuwC3idYMw90m3AJ8M5/b8Ojzt8kOAA8zaCoa3/BdI5OTcRHLDfBLwE3OruzwCYWXG4\nZ1EMED7+U+AFYHP4nMgv9k8TTDvdC9wCfNLdK8JtQwhmjx0gOD5zo7s/d5K1Sydh7rrIjkgkM3sY\nWO3u+u1XEo72FCThhUM3Q8wsKTzZ62LgsVjXJRILmn0kEpx38CjBeQplwLXhNFGRhKPhIxERaaLh\nIxERaRK14SMzG0Aw7a0PwXTD2e5+21FtLidYQ8YIToy51t2XHv1akfLz872kpCQqNYuIdFULFy7c\n5e4Fx2oXzWMK9cA33X1ROBd9oZk9H04/POId4Bx332tmFwCzCdaoaVFJSQmlpaXRq1pEpAsys01t\naRe1UHD37QRnWOLu+81sFVBEsALmkTbzI57yOtA/WvWIiMixdcgxBTMrIVgbZUErza4Cnm7h+TPN\nrNTMSisqKpprIiIi7SDqoWBmWQSnx9/g7vtaaHMeQSh8p7nt7j7b3Se7++SCgmMOiYmIyAmK6nkK\n4SX65gIPuPujLbQZB9xNcNGT3dGsR0REWhe1PYXwAiZ/AFa5+y9aaFNMcNLQ58JFuEREJIaiuadw\nJsEVrN4ysyXhY98juDIV7n4X8EOCs0jvCNe3r3f3yVGsSUREWhHN2Uev8O718JtrczVwdbRqEBGR\n45MwZzSv3rGPW59dTWVNbaxLERHptBImFDbuquG3L6xny56Dx24sIpKgEiYUeucE1y+pqD50jJYi\nIokrcUIhOwiF8n3Hc0EtEZHEkjChUHAkFPYrFEREWpIwoZCekkxuRirl+zV8JCLSkoQJBYCCrHQN\nH4mItCKhQqF3TrqGj0REWpFYoZDdjQqFgohIixIsFNKp2H8YXZdaRKR5CRUKBdnp1DY0UnWwLtal\niIh0SgkVCr1zugGalioi0pLECgWdwCYi0qqECoV/n8CmcxVERJqTUKHQW2c1i4i0KqFCISs9he6p\nyZqWKiLSgoQKBTPTCWwiIq1IqFCAYAipfJ+OKYiINCdqoWBmA8zsBTNbaWYrzGxWM23MzH5tZuvM\nbJmZTYpWPUforGYRkZZFc0+hHvimu48GzgCuN7PRR7W5ABgW3mYCd0axHiCYgaThIxGR5kUtFNx9\nu7svCn/eD6wCio5qdjFwvwdeB3LNrDBaNUEQCtWH66mprY/m24iIxKUOOaZgZiXARGDBUZuKgC0R\n98t4b3BgZjPNrNTMSisqKk6qliPTUjWEJCLyXlEPBTPLAuYCN7j7vhN5DXef7e6T3X1yQUHBSdWj\npS5ERFoW1VAws1SCQHjA3R9tpslWYEDE/f7hY1GjpS5ERFoWzdlHBvwBWOXuv2ih2RPAFeEspDOA\nKnffHq2aIPKsZk1LFRE5WkoUX/tM4HPAW2a2JHzse0AxgLvfBTwFXAisA2qAL0SxHgB6ZqSRkmQa\nPhIRaUbUQsHdXwHsGG0cuD5aNTQnKcmCaakaPhIReY+EO6MZgmmpFdUKBRGRoyVkKGipCxGR5iVk\nKBRoqQsRkWYlZCj0zk5n94Fa6hoaY12KiEinkpihkBNMS92l4woiIu+SmKGQHZ7VrBlIIiLvkqCh\noMtyiog0JyFDoUCL4omINCshQyE/S0tdiIg0JyFDIS0libzMNA0fiYgcJSFDAY6cwKZQEBGJlLCh\nUJCdToWGj0RE3iVhQ6F3djcNH4mIHCVxQyEnnV3Vh2ls9FiXIiLSaSRsKBRkpVPX4FQerIt1KSIi\nnUbChsKRpS40LVVE5N8SNxS01IWIyHskcChoqQsRkaNFLRTM7B4zKzez5S1s72FmfzezpWa2wsyi\nfn3mSBo+EhF5r2juKcwBZrSy/XpgpbuPB84Ffm5maVGs510y0lLISk/R+kciIhGiFgruPg/Y01oT\nINvMDMgK29ZHq57m9M5O1/CRiEiEWB5TuB0YBWwD3gJmuXuzl0Izs5lmVmpmpRUVFe1WQH52OhU6\n0Cwi0iSWofAhYAnQD5gA3G5mOc01dPfZ7j7Z3ScXFBS0WwHBnoKOKYiIHBHLUPgC8KgH1gHvACM7\nsgAtdSEi8m6xDIXNwHQAM+sDjAA2dGQBvXPSqaltoPpwhx7KEBHptFKi9cJm9iDBrKJ8MysDbgJS\nAdz9LuDHwBwzewsw4Dvuvita9TSnd8QV2LLSo/ZXISISN6L2Tejulx1j+zbgg9F6/7b491nNhxiU\nnxnLUkREOoWEPaMZoG+PIBQ276mJcSUiIp1DQofCoPxMstNTWLylMtaliIh0CgkdCslJxsSBPVm4\ncW+sSxER6RQSOhQATi3uydry/VTpugoiIgqFySU9cYfFm7W3ICKS8KEwfkAuSQaLNikUREQSPhSy\n0lMY2TeHhdpTEBFRKEAwhLR4cyX1Dc2uxycikjAUCsCpA3tSU9vA6h37Y12KiEhMKRQIQgFgoY4r\niEiCUygARbnd6ZOTrlAQkYSnUADMjFMH9lQoiEjCUyiETh2Yx9bKg+yo0kV3RCRxKRRCOq4gIqJQ\naHJKvxy6pSZRumlPrEsREYkZhUIoNTmJcf1zdWaziCQ0hUKEUwf2ZMW2fRysbYh1KSIiMaFQiDB5\nYE/qG52lZbq+gogkpqiFgpndY2blZra8lTbnmtkSM1thZi9Fq5a2mlSsg80iktiiuacwB5jR0kYz\nywXuAD7q7qcAn4piLW3SMzONIQWZCgURSVhRCwV3nwe0NpXnM8Cj7r45bF8erVqOx6kDe7Jo814a\nGz3WpYiIdLhYHlMYDvQ0sxfNbKGZXdFSQzObaWalZlZaUVER1aImD8yjsqaODbsORPV9REQ6o1iG\nQgpwKvBh4EPAD8xseHMN3X22u09298kFBQVRLWpSeBJb6UadryAiiSeWoVAGPOvuB9x9FzAPGB/D\negAYUpBJn5x0XljTKUazREQ6VCxD4XHgLDNLMbMM4HRgVQzrAYLF8Wac0pcX11Rw4HB9rMsREelQ\n0ZyS+iDwGjDCzMrM7Cozu8bMrgFw91XAM8Ay4A3gbndvcfpqR5oxppDD9Y28uCa6xy9ERDqblGi9\nsLtf1oY2twK3RquGE3XaoDx6Zabx1PLtfHhcYazLERHpMDqjuRnJScYHT+nLC6vLOVSnJS9EJHEo\nFFpw4di+1NQ2MG+thpBEJHEoFFpwxuBe9OieytPLd8S6FBGRDqNQaEFqchIfHN2Hf67ayeF6DSGJ\nSGJQKLTigrF92X+onvnrdse6FBGRDqFQaMWZQ/PJTk/h6eXbY12KiEiHUCi0Ij0lmemjevPcyp3U\nNTTGuhwRkahTKBzDBWMLqaypY8EGrYUkIl2fQuEYzhleQEZaMk9pCElEEoBC4Ri6pSZz3sjePLdi\nBw26xoKIdHEKhTa4YExfdlXXajltEeny2hQKZvaeS2U291hXdd6I3qSnJPH40m2xLkVEJKrauqfw\n3TY+1iVlpqfw8YlFPFJaRtnemliXIyISNa2GgpldYGa/AYrM7NcRtzlAQl1s4GvTh4HBbf98O9al\niIhEzbH2FLYBpcAhYGHE7QmCS2gmjH653bnijIHMXVTG2zv3x7ocEZGoaDUU3H2pu98HDHX3+8Kf\nnwDWufveDqmwE7nuvKFkpKXw8+fWxroUEZGoaOsxhefNLMfM8oBFwO/N7JdRrKtTystM40vTBvPM\nih0s3VIZ63JERNpdW0Ohh7vvAy4B7nf304Hp0Sur87pq2iB6Zabx02dXx7oUEZF219ZQSDGzQuBS\n4Mm2PMHM7jGzcjNr9brLZjbFzOrN7JNtrCWmstJTuP68oby6bjevvL0r1uWIiLSrtobCfwHPAuvd\n/U0zGwwcaxrOHGBGaw3MLBn4X+C5NtbRKVx+RjFFud259dnVuOssZxHpOtoUCu7+V3cf5+7Xhvc3\nuPsnjvGcecCxTgH+KjAXKG9LHZ1FekoyN5w/jKVlVTy7QldmE5Guo61nNPc3s7+Fw0HlZjbXzPqf\nzBubWRHwceDONrSdaWalZlZaUdE5rpl8yaT+DO2dxf88vZqa2oQ6ZUNEurC2Dh/dSzAVtV94+3v4\n2Mn4FfAddz/mhQrcfba7T3b3yQUFBSf5tu0jOcn4ycfGsHlPDf/zlA46i0jX0NZQKHD3e929PrzN\nAU7223ky8JCZbQQ+CdxhZh87ydfsUGcM7sVVZw7ij69v4qW1nWMPRkTkZLQ1FHab2WfNLDm8fRY4\nqQsXu/sgdy9x9xLgEeA6d3/sZF4zFr71oREM653Ftx9ZSmVNbazLERE5KW0NhS8STEfdAWwn+M3+\nytaeYGYPAq8BI8yszMyuMrNrzOyak6i30+mWmswv/2MCu6tr+eHjK2JdjojISUlpY7v/Aj5/ZGmL\n8MzmnxGERbPc/bK2FuHuV7a1bWc0pqgHs6YP4+fPr+UDo/vwkfH9Yl2SiMgJaeuewrjItY7cfQ8w\nMTolxadrzx3C+AG5/ODx5ezcdyjW5YiInJC2hkKSmfU8cifcU2jrXkZCSElO4heXjudQXQNff3gJ\nVTV1sS5JROS4tTUUfg68ZmY/NrMfA/OBn0avrPg0pCCL/7p4DAve2cMHfvkSz6/cGeuSRESOS1vP\naL6fYDG8neHtEnf/YzQLi1eXTh7AY9edGayoen8psx5azJ4DmpUkIvHB4m3tnsmTJ3tpaWmsyzim\n2vpG7nxxPbe/8DY53VL58cfGcOHYwliXJSIJyswWuvvkY7Vr6/CRHKe0lCRmnT+Mv3/1LPrldue6\nBxZpOElEOj2FQpSN7JvDI9dOZXRhDt+Zu4zy/ZqZJCKdl0KhA6SnJHPbpydw4HA933lkmZbbFpFO\nS6HQQYb1yeZ7F47ihTUV/On1TbEuR0SkWQqFDnTF1IGcM7yAn/xjFevK98e6HBGR91AodCAz49ZP\njSMzPYVZDy2htv6Yq4aLiHQohUIH653djVsuGcuKbfv4xfNrY12OiMi7KBRi4IOn9OWy0wbwu3nr\nufvlDTQ26sCziHQOCoUY+cFFo5k+sg8/+ccqLr97AVsrD8a6JBERhUKsZKSl8PsrTuWnnxjHsrJK\nZvxyHo8uKtN0VRGJKYVCDJkZl04ZwNOzzmZkYTbf+MtSrntgEbuqD8e6NBFJUAqFTqC4VwYPzZzK\njReM5J+rdvL+n73IH1/bSIOONYhIB1ModBLJScY15wzh6VnTGFPUgx88voKLf/sKizbvPfaTRUTa\nSdRCwczuMbNyM1vewvbLzWyZmb1lZvPNbHy0aoknQ3tn88DVp/ObyyZSsf8wl9wxn+88soy9Wn5b\nRDpANPcU5gAzWtn+DnCOu48FfgzMjmItccXM+Mj4fvzrm+cy8+zBzF1UxiV3zqdsb02sSxORLi5q\noeDu84A9rWyfH3Hd59eB/tGqJV5lpafwvQtH8fCXz2BX9WEuves11ldUx7osEenCOssxhauAp1va\naGYzzazUzEorKio6sKzO4dSBeTw8cyq1DY1cetdrLN9aFeuSRKSLinkomNl5BKHwnZbauPtsd5/s\n7pMLCgo6rrhOZHS/HP7y5amkpyRx2ezXeXNjizthIiInLKahYGbjgLuBi919dyxriQeDC7J45Nr3\nUZCTzuf+sIB/rdKV3ESkfcUsFMysGHgU+Jy7a2W4NuqX252/fHkqQwqyuOq+Un70xAoO1jbEuiwR\n6SKiOSX1QeA1YISZlZnZVWZ2jZldEzb5IdALuMPMlphZabRq6Wrys9KZe+37+MKZJcyZv5GLfvMy\ny8oqY12WiHQBFm9r7UyePNlLS5UfR7zy9i6+9del7Ko+zNemD+O6c4eQkhzzQ0Ui0smY2UJ3n3ys\ndvr2iHNnDcvn2RvO5sKxhfzi+bV86nevsWn3gViXJSJxSqHQBfTISOXXl03k15dNZF15NRfe9jKP\nLNSKqyJy/BQKXchHx/fjmRvO5pSiHnzrr0v5yoOLqaqpi3VZIhJHFApdTFFudx780hl8e8YInl2+\ngxm3zeO19ZrtKyJto1DogpKTjOvOHcqj172PbqnJfObu1/nvp1ZxqE5TV0WkdQqFLmxc/1z+8bWz\n+Mxpxcyet4GLb3+Vldv2xbosEenEFApdXEZaCjd/fCz3XjmFPTW1XPzbV7jjxXW6gI+INEuhkCDO\nG9mbZ284mw+M7sNPn1nDpb97jXlrK2hUOIhIBJ28lmDcnceWbOUnT65i94FaivMyuOy0Yj41uT/5\nWemxLk9EoqStJ68pFBLU4foGnlm+gz8v2MyCd/aQmmzMGFPINz8wnJL8zFiXJyLtTKEgbbaufD8P\nLNjMI6VlNLrz35eM5eIJRbEuS0TakZa5kDYb2jubmz5yCs98/WxG98th1kNL+P/+upSa2vpYlyYi\nHUyhIE2OnPj2tfcP5ZFFZVz0m1c0hVUkwWj4SJo1f/0ubnhoCZUH6/jAqD4Mys+kJD+TQfkZlPTK\nJC8zDTOLdZki0kZtHT5K6YhiJP68b0g+T8+axs1PrWLRpr08s2LHu85tGD8gl5s/NoYxRT1iWKWI\ntDftKUib1DU0Urb3IO/sqmbtzmrufnkDe2vquOqsQdxw/jAy0vT7hUhnptlHElWVNbXc8vRqHnpz\nC0W53fnJx8dw3ojesS5LRFqg2UcSVbkZadzyiXH85ctT6ZaaxBfufZPr/7yI7VUHY12aiJyEaF6j\n+R4zKzez5S1sNzP7tZmtM7NlZjYpWrVI9Jw2KI+nZk3j6+cP558rd/L+n73Eb19YpxVZReJUNPcU\n5gAzWtl+ATAsvM0E7oxiLRJF6SnJzDp/GP/8xjmcM7yAW59dw4d+NY9/rdoZ69JE5DhF7eigu88z\ns5JWmlwM3O/BQY3XzSzXzArdfXu0apLoGpCXwV2fO5WX367gP/++kqvuK2VKSU9G9s2hqGd3inK7\nU9SzOwPzMuildZZEOqVYThkpArZE3C8LH3tPKJjZTIK9CYqLizukODlx04YV8PSsadw3fyNzF23l\n8SVb2Xfo3WdHnzuigC+cOYhpQ/NJStL5DiKdRVzMI3T32cBsCGYfxbgcaYPU5CSunjaYq6cNBmD/\noTq2Vh5kW+VBlmyp4sE3NvP5e95gcEEmV76vhEsm9ScrPS4+jiJdWlSnpIbDR0+6+5hmtv0OeNHd\nHwzvrwHOPdbwkaakdg219Y089dZ27n31HZaWVZGRlkxhj25kpaeQkZZCZnoy2d1S+ewZAzl1YM9Y\nlysS9+LhjOYngK+Y2UPA6UCVjickjrSUJD42sYiPTSxi8ea9PLpoK3sO1FJ9uJ6a2nq2VdaxvWov\nTy/fzt1XTOGsYfmxLlkkIUQtFMzsQeBcIN/MyoCbgFQAd78LeAq4EFgH1ABfiFYt0rlNLO7JxOL3\n7g3sqj7MZ+9ewBfve5M7L5/E9FF9YlCdSGLRGc3SqVXW1HLFPW+wcts+fn3ZRC4cWxjrkkTiks5o\nli4hNyONP119OhMG5PKVPy/ib4vLmrY1NDrbqw5SunEPZXtrYlilSNehPQWJCwcO1/Ol+0t5bcNu\npgzMY8e+Q2yvOkhdQ/D57ZaaxK2fHM9HxveLcaUinVM8HGgWabPM9BTuuXIKP3hsORt2HWDCgFw+\nPK6QotzuFPboxl0vreerDy5mzY79fOMDw3Xug8gJUihI3OiWmsytnxrf7LZpwwr4wWPLuf2Fdaze\nsZ9ffXqCznsQOQH6XyNdQlpKErd8YiyjCrP58T9Wcckdr3LH5ZPo26M7SQZJZpiBOxysbaCmroGD\ntfXU1DbQ6DCuqIf2LkRQKEgXYmZceeYghvbO5vo/L+L8X8xr83NPG5THrZ8cx8BemVGsUKTz04Fm\n6ZK27KnhuZU7aWx0Gt1pdGgMP+sZaclkpCXTPS2FzLRktlUe5KfPrKG+0fnuhSP57OkDtdcgXY6u\nvCZyHLZXHeTGuW/x0toKpg7uxU8/OY4BeRmxLkuk3SgURI6Tu/OX0i38+MlVuDszxhRy6sCenDqw\nJ8N6Z2nvQeKapqSKHCcz4z+mFHPWsAL+9+nVvLimnLmLgpPlsrulMLG4J6cPyuOc4QWMLsxRSEiX\npD0FkRa4O5v31LBw096m2+od+wHIz0pj2rACzh6ez7nDe9MzMy3G1Yq0TnsKIifJzBjYK5OBvTK5\nZFJ/AMr3H+LltbuY93YFL62t4G+Lt9ItNYlPTynm6mmD6N9TxyEkvmlPQeQENTY6b22t4k+vb+Jv\ni7fiwEfH9+Oac4Ywom92rMsTeRcdaBbpQNsqD/KHV97hwTc2U1PbwLj+PchKTyEtJYnU5CTSUpLo\nlpJMbkYqud1Tyc1IpUdGGplpyVTW1LGr+nB4q6XqYB2nDcrjE5P6U5Cta1lL+1AoiMRAZU0tf3xt\nE6+/s5va+kZq6xs5XN9IXUMjh+oaqTpYR/Xh+mafm56SRH5WOt3TkllXXk1KkjF9VG8+PaWYs4cX\nkKwD23ISFAoinVRdQxAOlTVBQOR2TyU/O53MtGTMgi/+deXV/KV0C3MXlrH7QC2FPbpx+enFfO6M\nEnpkpMa4BxKPFAoiXUBtfSP/t3onDyzYzMtv7yIjLZnLTivmqrMG0S+3e6zLkziiUBDpYlZt38fs\neRt4Yuk2DPjohH5cdlox4/r3ID0lOdblSSfXKULBzGYAtwHJwN3ufstR24uB+4DcsM2N7v5Ua6+p\nUJBEV7a3hrtffoeH39zCwboG0lKSmNA/lymDejK5JI+JA3LJzdB5E/JuMQ8FM0sG1gIfAMqAN4HL\n3H1lRJvZwGJ3v9PMRgNPuXtJa6+rUBAJVNXU8dqG3ZRu3MObG/ewfNs+GhqD/8+9MtMYUpDF4IJM\nBhdkMqxPNqcO7ElONx2PSFSd4eS104B17r4hLOgh4GJgZUQbB3LCn3sA26JYj0iX0iMjlRlj+jJj\nTF8AamrrWby5khXbqthQcYD1FdU8v3Inuw/UApBkMKowh9MH9eK0QXmcNiiPPJ2JLUeJZigUAVsi\n7pcBpx/V5kfAc2b2VSATOD+K9Yh0aRlpKZw5NJ8zh+a/6/HKmlpWbtvHGxv3sGDDHh5YsIl7Xn0H\ngLFFPThneAHnjChg4oBcUpKTYlG6dCKxXubiMmCOu//czKYCfzSzMe7eGNnIzGYCMwGKi4tjUKZI\n/MrNSON9Q/N5XxgWtfWNvLW1kvnrdjPv7QrufGk9t7+wjuxuKZw5JJ/xA3IZWZjN6MIcemenY2a4\nOxt31/Da+t28vmE3b27cw/i9Z7/FAAALR0lEQVT+ufzs0vG67GkXE81jClOBH7n7h8L73wVw9/+J\naLMCmOHuW8L7G4Az3L28pdfVMQWR9lV1sI7564L1nOat3cXWyoNN23pmpDKsdzab99SwY98hAAqy\n0xnfP5cX1pQzok8291w5hb49usWqfGmjznCgOYXgQPN0YCvBgebPuPuKiDZPAw+7+xwzGwX8Cyjy\nVopSKIhEV1VNHat37GP1jv2s3rGPtTurKezRjTMG92LqkF4Mzs/EzHhxTTnXP7CInO6p3PuFKYzs\nm/Oe11pfUc2iTXupbWikvsGpa2ikrsFJT0nionGF9M5RmHSUmIdCWMSFwK8Ippve4+43m9l/AaXu\n/kQ44+j3QBbBQedvu/tzrb2mQkGk81ixrYovznmTA4cbuPOzk5g2rIDd1Yf5+9Jt/G3xVpaWVbX4\n3NRk48NjC7nyzEFMGJDbgVUnpk4RCtGgUBDpXLZVHuSLc95kXXk1pw/OY8GGPdQ3OqMKc7hkYhHv\nH9WbzLQUUpON1JQkUpOS2LHvEPe/tpG/lpZRfbieicW5XPm+EiYV96Rfbnet8xQFCgUR6TD7D9Xx\nzb8sZeX2fXx4bCEfn1TU7HBSc897ZGEZ983fyMbdNQCkJScxIK87Jb0yGZCXgbsHa0WF60XtO1TH\nmUPyufGCkWTqIHebKRREJG40NjqLNu9lXXk1G3fXsHHXATbuPsCWPTUkJxm5GWn0CJccT01O4oU1\n5RTnZfCLSydw6sCeJ/Xe9eEChb2yuvYy5Z3h5DURkTZJSjIml+QxuSSvTe0XbNjNN/6ylE/dNZ/r\nzxvK16YPI/UY51g0NjobdlWzfOs+1pVXs76iOgyhA9Q1OBeM6cv3Pzwq4a+epz0FEYlL+w/V8aMn\nVjJ3URlji3owa/ow0lOTSDLDDAxjb00tS8sqWbalire2VjVdyyI5yRiYl8GQ3lkMKcjCDOa8uhHH\nufacoXz5nMF0S215kcHD9Q1s2XOQzXsOsHl3DcP6ZL/npMHORsNHIpIQnn5rO9/721vsralrdntq\nsjGqMIfx/XMZ178H4/rnUpKf8Z6VZbdVHuTmp1bxj2XbGZDXne9fOJp+ud3YtLuGzXtq2Ly7hk1h\nCGzfd4ijvzovmVTETRed0mmvd6FQEJGEUVVTx9vl+3GCYaJGB8fJSk9hRN/s41pafP66Xdz0xAre\nLq9+1+P5WekM7JXBwLwMintlMLBXBsV5GRTlZvDnBZv47Yvr6ZWZxv9cMpbpo/q0cw9PnkJBROQE\n1TU08uyKHaQkJTV9+R9rptPyrVV8669LWb1jf5v2Gg7XN/DmO3tZub2KM4fmM7owp+nKe9GgUBAR\n6WC19Y385v/e5o4X15NsxsjCbE7p14OxRT0YU5RDbvc0Xl5XwQurK5i/fhc1tQ1Nzx3eJ4uLJxRx\n8YR+UTnYrVAQEYmRFduqeHzJNpZvrWL51ir2Hap/1/b+Pbtz3ojenDuigFGFOfzf6nIeW7yV0k17\nATitJI+JA3MZWpDF0N7BLfskr4WhUBAR6QTcnbK9B1m+tYpdB2qZOrgXQwoymx0q2ry7hseXbOWp\n5TtYV76fuoZ/fz/3yUnnS9MGc/W0wSdUh85TEBHpBMyMAXkZDMg79pBQca8Mvjp9GF+dPoy6hkY2\n76lhXXlwPsX68moKsqN/gp1CQUSkE0pNTmJIQXAexYdO6bj31WWWRESkiUJBRESaKBRERKSJQkFE\nRJooFEREpIlCQUREmigURESkiUJBRESaxN0yF2ZWAWw6wafnA7vasZxYU386r67UF+ha/elKfYG2\n92eguxccq1HchcLJMLPStqz9ES/Un86rK/UFulZ/ulJfoP37o+EjERFpolAQEZEmiRYKs2NdQDtT\nfzqvrtQX6Fr96Up9gXbuT0IdUxARkdYl2p6CiIi0QqEgIiJNEiYUzGyGma0xs3VmdmOs6zleZnaP\nmZWb2fKIx/LM7Hkzezv8s2csa2wrMxtgZi+Y2UozW2Fms8LH47U/3czsDTNbGvbnP8PHB5nZgvAz\n97CZpcW61rYys2QzW2xmT4b347kvG83sLTNbYmal4WPx+lnLNbNHzGy1ma0ys6nt3ZeECAUzSwZ+\nC1wAjAYuM7PRsa3quM0BZhz12I3Av9x9GPCv8H48qAe+6e6jgTOA68N/j3jtz2Hg/e4+HpgAzDCz\nM4D/BX7p7kOBvcBVMazxeM0CVkXcj+e+AJzn7hMi5vPH62ftNuAZdx8JjCf4N2rfvrh7l78BU4Fn\nI+5/F/hurOs6gX6UAMsj7q8BCsOfC4E1sa7xBPv1OPCBrtAfIANYBJxOcJZpSvj4uz6DnfkG9A+/\nXN4PPAlYvPYlrHcjkH/UY3H3WQN6AO8QThCKVl8SYk8BKAK2RNwvCx+Ld33cfXv48w6gTyyLORFm\nVgJMBBYQx/0Jh1uWAOXA88B6oNLd68Mm8fSZ+xXwbaAxvN+L+O0LgAPPmdlCM5sZPhaPn7VBQAVw\nbzi0d7eZZdLOfUmUUOjyPPg1Ia7mF5tZFjAXuMHd90Vui7f+uHuDu08g+C37NGBkjEs6IWZ2EVDu\n7gtjXUs7OsvdJxEMH19vZmdHboyjz1oKMAm4090nAgc4aqioPfqSKKGwFRgQcb9/+Fi822lmhQDh\nn+UxrqfNzCyVIBAecPdHw4fjtj9HuHsl8ALBEEuumaWEm+LlM3cm8FEz2wg8RDCEdBvx2RcA3H1r\n+Gc58DeC0I7Hz1oZUObuC8L7jxCERLv2JVFC4U1gWDiDIg34NPBEjGtqD08Anw9//jzB2HynZ2YG\n/AFY5e6/iNgUr/0pMLPc8OfuBMdHVhGEwyfDZnHRH3f/rrv3d/cSgv8n/+fulxOHfQEws0wzyz7y\nM/BBYDlx+Flz9x3AFjMbET40HVhJe/cl1gdPOvAgzYXAWoKx3u/Hup4TqP9BYDtQR/Abw1UEY73/\nAt4G/gnkxbrONvblLIJd3GXAkvB2YRz3ZxywOOzPcuCH4eODgTeAdcBfgfRY13qc/ToXeDKe+xLW\nvTS8rTjyfz+OP2sTgNLws/YY0LO9+6JlLkREpEmiDB+JiEgbKBRERKSJQkFERJooFEREpIlCQURE\nmigUJOGY2fzwzxIz+0w7v/b3mnsvkXihKamSsMzsXOBb7n7RcTwnxf+9BlBz26vdPas96hOJBe0p\nSMIxs+rwx1uAaeE6+18PF7W71czeNLNlZvblsP25ZvaymT1BcAYpZvZYuMDaiiOLrJnZLUD38PUe\niHwvC9xqZsvDtf3/I+K1X4xYI/+B8IxvzOwWC645sczMftaRf0eSuFKO3USky7qRiD2F8Mu9yt2n\nmFk68KqZPRe2nQSMcfd3wvtfdPc94bIWb5rZXHe/0cy+4sHCeEe7hOBs1PFAfviceeG2icApwDbg\nVeBMM1sFfBwY6e5+ZBkNkWjTnoLIv30QuCJcAnsBwfIBw8Jtb0QEAsDXzGwp8DrBYovDaN1ZwIMe\nrKa6E3gJmBLx2mXu3kiw5EcJUAUcAv5gZpcANSfdO5E2UCiI/JsBX/XgCl0T3H2Qux/ZUzjQ1Cg4\nFnE+MNWDq60tBrqdxPsejvi5geBiNvUEq3k+AlwEPHMSry/SZgoFSWT7geyI+88C14bLemNmw8OV\nNY/WA9jr7jVmNpLgkqJH1B15/lFeBv4jPG5RAJxNsMBcs8JrTfRw96eArxMMO4lEnY4pSCJbBjSE\nw0BzCK4bUAIsCg/2VgAfa+Z5zwDXhOP+awiGkI6YDSwzs0UeLDl9xN8IrrGwlGCF2G+7+44wVJqT\nDTxuZt0I9mC+cWJdFDk+mpIqIiJNNHwkIiJNFAoiItJEoSAiIk0UCiIi0kShICIiTRQKIiLSRKEg\nIiJN/n9wT2fe7ZefeQAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "metadata": {
        "colab_type": "code",
        "id": "oRexN3ZjbfiA",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}