3b_MLP_MNIST_Modern_160517.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\\* *[Notice] I wrote thie code while following the examples in [Choi's Tesorflow-101 tutorial](https://github.com/sjchoi86/Tensorflow-101). And,  as I know, most of Choi's examples originally come from [Aymeric Damien's](https://github.com/aymericdamien/TensorFlow-Examples/) and  [Nathan Lintz's ](https://github.com/nlintz/TensorFlow-Tutorials) tutorials.*"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. Multilayer Perceptron with MNIST data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "import matplotlib.pyplot as plt\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "#%matplotlib inline  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load MNIST data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting data/train-images-idx3-ubyte.gz\n",
      "Extracting data/train-labels-idx1-ubyte.gz\n",
      "Extracting data/t10k-images-idx3-ubyte.gz\n",
      "Extracting data/t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "mnist      = input_data.read_data_sets('data', one_hot=True)\n",
    "X_train   = mnist.train.images\n",
    "Y_train = mnist.train.labels\n",
    "X_test    = mnist.test.images\n",
    "Y_test  = mnist.test.labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape of (X_train, X_test, Y_train, Y_test)\n",
      "((55000, 784), (10000, 784), (55000, 10), (10000, 10))\n"
     ]
    }
   ],
   "source": [
    "dimX = X_train.shape[1]\n",
    "dimY = Y_train.shape[1]\n",
    "nTrain = X_train.shape[0]\n",
    "nTest = X_test.shape[0]\n",
    "print (\"Shape of (X_train, X_test, Y_train, Y_test)\")\n",
    "print (X_train.shape, X_test.shape, Y_train.shape, Y_test.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Xavier's initialize method\n",
    "X. Glorot and Y. Bengio, \"Understanding the difficulty of training deep feedforward neural networks\", 2010."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def xavier_init(n_inputs, n_outputs, uniform=True):\n",
    "    if uniform:\n",
    "        init_range = tf.sqrt(6.0 / (n_inputs + n_outputs))\n",
    "        return tf.random_uniform_initializer(-init_range, init_range)\n",
    "    else:\n",
    "        stddev = tf.sqrt(3.0 / (n_inputs + n_outputs))\n",
    "    return tf.truncated_normal_initializer(stddev=stddev)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define my neural network structure"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "nLayer0 = dimX\n",
    "nLayer1 = 256\n",
    "nLayer2 = 256\n",
    "nLayer3 = 256\n",
    "nLayer4 = 256\n",
    "nLayer5 =  dimY\n",
    "sigma_init = 0.1   # For randomized initialization\n",
    "myDropProb = 0.7 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "W = {\n",
    "    'W1': tf.get_variable(\"W1\", shape=[nLayer0, nLayer1], initializer=xavier_init(nLayer0,nLayer1)),\n",
    "    'W2': tf.get_variable(\"W2\", shape=[nLayer1, nLayer2], initializer=xavier_init(nLayer1,nLayer2)),\n",
    "    'W3': tf.get_variable(\"W3\", shape=[nLayer2, nLayer3], initializer=xavier_init(nLayer2,nLayer3)),\n",
    "    'W4': tf.get_variable(\"W4\", shape=[nLayer3, nLayer4], initializer=xavier_init(nLayer3,nLayer4)),\n",
    "    'W5': tf.get_variable(\"W5\", shape=[nLayer4, nLayer5], initializer=xavier_init(nLayer4,nLayer5))\n",
    "}\n",
    "b = {\n",
    "    'b1': tf.Variable(tf.random_normal([nLayer1])),\n",
    "    'b2': tf.Variable(tf.random_normal([nLayer2])),\n",
    "    'b3': tf.Variable(tf.random_normal([nLayer3])),\n",
    "    'b4': tf.Variable(tf.random_normal([nLayer4])),\n",
    "    'b5': tf.Variable(tf.random_normal([nLayer5]))\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def model_myNN(_X, _W, _b, _dropout_prob):\n",
    "    Layer1 = tf.nn.dropout(tf.nn.relu(tf.add(tf.matmul(_X,_W['W1']), _b['b1'])), _dropout_prob)\n",
    "    Layer2 = tf.nn.dropout(tf.nn.relu(tf.add(tf.matmul(Layer1,_W['W2']), _b['b2'])), _dropout_prob)\n",
    "    Layer3 = tf.nn.dropout(tf.nn.relu(tf.add(tf.matmul(Layer2,_W['W3']), _b['b3'])), _dropout_prob)\n",
    "    Layer4 = tf.nn.dropout(tf.nn.relu(tf.add(tf.matmul(Layer3,_W['W4']), _b['b4'])), _dropout_prob)\n",
    "    Layer5 = tf.add(tf.matmul(Layer4,_W['W5']), _b['b5'])                       \n",
    "    return Layer5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X = tf.placeholder(tf.float32, [None, dimX], name=\"input\")\n",
    "Y= tf.placeholder(tf.float32, [None, dimY], name=\"output\")\n",
    "dropout_prob = tf.placeholder(tf.float32, name=\"dropout\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "Y_pred = model_myNN(X, W, b, dropout_prob)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## Define loss function, optimizer, measurer"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We use *softmax_cross_entropy()* and *AdamOptimizer()*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(Y_pred, Y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "learning_rate = 0.001\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(loss)\n",
    "#optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(loss)\n",
    "training_epochs = 30\n",
    "display_epoch = 5\n",
    "batch_size = 100   # For each time, we will use 100 samples to update parameters "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "correct_prediction = tf.equal(tf.argmax(Y_pred, 1), tf.argmax(Y, 1))    \n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, \"float\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Run the session"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "We use *with* for load a TF session"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(epoch 5)\n",
      "[Loss / Tranining Accuracy] 0.0538 / 0.9837\n",
      " \n",
      "(epoch 10)\n",
      "[Loss / Tranining Accuracy] 0.0306 / 0.9902\n",
      " \n",
      "(epoch 15)\n",
      "[Loss / Tranining Accuracy] 0.0174 / 0.9949\n",
      " \n",
      "(epoch 20)\n",
      "[Loss / Tranining Accuracy] 0.0120 / 0.9967\n",
      " \n",
      "(epoch 25)\n",
      "[Loss / Tranining Accuracy] 0.0094 / 0.9971\n",
      " \n",
      "(epoch 30)\n",
      "[Loss / Tranining Accuracy] 0.0088 / 0.9973\n",
      " \n",
      "[Test Accuracy]  0.9815\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "    sess.run(tf.initialize_all_variables())\n",
    "\n",
    "    for epoch in range(training_epochs):\n",
    "        nBatch  = int(nTrain/batch_size)\n",
    "        #myIdx =  np.random.permutation(nTrain)\n",
    "        for ii in range(nBatch):\n",
    "            X_batch, Y_batch = mnist.train.next_batch(batch_size)\n",
    "            #X_batch = X_train[myIdx[ii*batch_size:(ii+1)*batch_size],:]\n",
    "            #Y_batch = Y_train[myIdx[ii*batch_size:(ii+1)*batch_size],:]\n",
    "            sess.run(optimizer, feed_dict={X:X_batch, Y:Y_batch, dropout_prob:myDropProb})\n",
    "          \n",
    "        if (epoch+1) % display_epoch == 0:\n",
    "            loss_temp = sess.run(loss, feed_dict={X: X_train, Y:Y_train, dropout_prob:1.})\n",
    "            accuracy_temp = accuracy.eval({X: X_train, Y:Y_train, dropout_prob:1.})\n",
    "            print \"(epoch {})\".format(epoch+1) \n",
    "            print \"[Loss / Tranining Accuracy] {:05.4f} / {:05.4f}\".format(loss_temp, accuracy_temp)\n",
    "            print \" \"\n",
    "            \n",
    "    print \"[Test Accuracy] \",  accuracy.eval({X: X_test, Y: Y_test, dropout_prob:1.})   "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}