lecture12.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Load Packages\n",
    "==============="
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import os\n",
    "import copy\n",
    "import torch\n",
    "import codecs\n",
    "import numpy as np\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import matplotlib.pyplot as plt\n",
    "from torchvision import datasets\n",
    "\n",
    "print(torch.__version__) # This code has been updated for PyTorch 1.0.0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Load Data:\n",
    "==============="
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "download =  datasets.MNIST('./MNIST/', train=True, download=True) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Datapath = 'MNIST/raw/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_int(b):\n",
    "    return int(codecs.encode(b, 'hex'), 16)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def parse_byte(b):\n",
    "    if isinstance(b, str):\n",
    "        return ord(b)\n",
    "    return b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def read_image_file(path):\n",
    "    with open(path, 'rb') as f:\n",
    "        data = f.read()\n",
    "        assert get_int(data[:4]) == 2051\n",
    "        length = get_int(data[4:8])\n",
    "        num_rows = get_int(data[8:12])\n",
    "        num_cols = get_int(data[12:16])\n",
    "        images = []\n",
    "        idx = 16\n",
    "        for l in range(length):\n",
    "            img = []\n",
    "            images.append(img)\n",
    "            for r in range(num_rows):\n",
    "                row = []\n",
    "                img.append(row)\n",
    "                for c in range(num_cols):\n",
    "                    row.append(parse_byte(data[idx]))\n",
    "                    idx += 1\n",
    "        assert len(images) == length\n",
    "        return torch.ByteTensor(images).view(-1,784)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def read_label_file(path):\n",
    "    with open(path, 'rb') as f:\n",
    "        data = f.read()\n",
    "        assert get_int(data[:4]) == 2049\n",
    "        length = get_int(data[4:8])\n",
    "        labels = [parse_byte(b) for b in data[8:]]\n",
    "        assert len(labels) == length\n",
    "        return torch.LongTensor(labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "TrainImages = read_image_file(os.path.join(Datapath, 'train-images-idx3-ubyte'))\n",
    "TrainLabels = read_label_file(os.path.join(Datapath, 'train-labels-idx1-ubyte'))\n",
    "TestImages = read_image_file(os.path.join(Datapath, 't10k-images-idx3-ubyte'))\n",
    "TestLabels = read_label_file(os.path.join(Datapath, 't10k-labels-idx1-ubyte'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "print(TrainImages.size())\n",
    "print(TrainLabels.size())\n",
    "print(TestImages.size())\n",
    "print(TestLabels.size())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Checking availability of GPU\n",
    "use_gpu = torch.cuda.is_available()\n",
    "if use_gpu:\n",
    "    print('GPU is available!')\n",
    "    device = \"cuda\"    \n",
    "else:\n",
    "    print('GPU is not available!')\n",
    "    device = \"cpu\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Define the Autoencoder:\n",
    "==============="
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class autoencoder(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(autoencoder, self).__init__()\n",
    "        self.encoder = nn.Sequential(\n",
    "            nn.Linear(28*28, 100),\n",
    "            nn.ReLU())\n",
    "        self.decoder = nn.Sequential(\n",
    "            nn.Linear(100, 28*28),\n",
    "            nn.ReLU())\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.encoder(x)\n",
    "        x = self.decoder(x)\n",
    "        return x\n",
    "\n",
    "\n",
    "net = autoencoder()\n",
    "print(net)\n",
    "\n",
    "if use_gpu:\n",
    "    net = net.double().cuda()\n",
    "else:\n",
    "    net = net.double()\n",
    "        \n",
    "init_weights = copy.deepcopy(net.encoder[0].weight.data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Define Optimization Technique:\n",
    "==============="
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "criterion = nn.MSELoss()\n",
    "optimizer = optim.SGD(net.parameters(), lr=0.5, momentum=0.9)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Train Autoencoder:\n",
    "==========="
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "iterations = 30\n",
    "BatchSize = 1000\n",
    "for epoch in range(iterations):\n",
    "    runningLoss = 0\n",
    "    for i in range(int(TrainImages.shape[0]/BatchSize)):\n",
    "        inputs = torch.index_select(TrainImages,0,torch.linspace(i*BatchSize,(i+1)*BatchSize - 1,steps=BatchSize)\n",
    "                                  .long()).double()\n",
    "        inputs = inputs/255\n",
    "        inputs = inputs.to(device)\n",
    "        \n",
    "        optimizer.zero_grad()\n",
    "        outputs = net(inputs)\n",
    "        loss = criterion(outputs, inputs)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        runningLoss += loss.item()\n",
    "    print('At Iteration : %d / %d  ;  Mean-Squared Error : %f'%(epoch + 1,iterations,runningLoss/\n",
    "                                                                (TrainImages.shape[0]/BatchSize)))\n",
    "print('Finished Training')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Testing Autoencoder Performance:\n",
    "================="
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "TestImg = torch.index_select(TestImages,0,torch.LongTensor([1]))\n",
    "\n",
    "outputImg = net((TestImg.double().to(device))/255).data\n",
    "outputImg = (outputImg*255).byte()\n",
    "outputImg = outputImg.view(-1,28,28)\n",
    "if device == 'cuda':\n",
    "    outputImg = outputImg.cpu()\n",
    "\n",
    "\n",
    "TestImg = TestImg.view(-1,28,28)\n",
    "\n",
    "fig = plt.figure()\n",
    "plot=fig.add_subplot(1,2,1)\n",
    "img = np.array(TestImg.numpy())[0]\n",
    "plot.set_title('Original Image')\n",
    "imgplot = plt.imshow(img,cmap='gray')\n",
    "\n",
    "plot=fig.add_subplot(1,2,2)\n",
    "img = np.array(outputImg.numpy())[0]\n",
    "plot.set_title('Reconstructed Image')\n",
    "imgplot = plt.imshow(img,cmap='gray')\n",
    "plt.show()\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Encoder Weights Visualization:\n",
    "======================="
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "trained_weights = copy.deepcopy(net.encoder[0].weight.data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "init_weights = (1 + init_weights)*127.5\n",
    "trained_weights = (1 + trained_weights)*127.5\n",
    "\n",
    "\n",
    "if use_gpu:\n",
    "    init_weights = init_weights.view(-1,280,280).byte().cpu()\n",
    "    trained_weights = trained_weights.view(-1,280,280).byte().cpu()\n",
    "else:\n",
    "    init_weights = init_weights.view(-1,280,280).byte()\n",
    "    trained_weights = trained_weights.view(-1,280,280).byte()\n",
    "\n",
    "d_weights = init_weights - trained_weights \n",
    "\n",
    "fig = plt.figure()\n",
    "plot=fig.add_subplot(1,3,1)\n",
    "img = np.array(init_weights.numpy())[0]\n",
    "plot.set_title('Initial Weights')\n",
    "imgplot = plt.imshow(img,cmap='gray')\n",
    "\n",
    "plot=fig.add_subplot(1,3,2)\n",
    "img = np.array(trained_weights.numpy())[0]\n",
    "plot.set_title('Trained Weights')\n",
    "imgplot = plt.imshow(img,cmap='gray')\n",
    "\n",
    "plot=fig.add_subplot(1,3,3)\n",
    "img = np.array(d_weights.numpy())[0]\n",
    "plot.set_title('Weight update')\n",
    "imgplot = plt.imshow(img,cmap='gray')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Remove Decoder and Add Classification Layer: \n",
    "================================"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_classifier = nn.Sequential(*list(net.children())[:-1])\n",
    "net = new_classifier\n",
    "net.add_module('classifier', nn.Sequential(nn.Linear(100, 10),nn.LogSoftmax(dim=1)))\n",
    "print(net)\n",
    "net = net.double().to(device)\n",
    "cll_weights = copy.deepcopy(net[0][0].weight.data)\n",
    "init_classifier_weights = copy.deepcopy(net.classifier[0].weight.data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Define Optimizer:\n",
    "================================"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "criterion = nn.NLLLoss()\n",
    "optimizer = optim.SGD(net.parameters(), lr=0.01, momentum=0.9)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Train Classifier:\n",
    "==========="
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "iterations = 30\n",
    "BatchSize = 1000\n",
    "for epoch in range(iterations):\n",
    "    runningLoss = 0\n",
    "    for i in range(int(TrainImages.shape[0]/BatchSize)):\n",
    "        inputs = torch.index_select(TrainImages,0,torch.linspace(i*BatchSize,(i+1)*BatchSize - 1,steps=BatchSize)\n",
    "                                  .long()).double()\n",
    "        labels = torch.index_select(TrainLabels,0,torch.linspace(i*BatchSize,(i+1)*BatchSize - 1,steps=BatchSize)\n",
    "                                  .long()).long()\n",
    "        inputs = inputs/255\n",
    "        inputs, labels = inputs.to(device), labels.to(device)\n",
    "        \n",
    "        optimizer.zero_grad()\n",
    "        outputs = net(inputs)\n",
    "        loss = criterion(outputs, labels)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        runningLoss += loss.item()\n",
    "        \n",
    "    inputs = TestImages.double()/255    \n",
    "    inputs = inputs.to(device)\n",
    "    outputs = net(inputs)\n",
    "    _, predicted = torch.max(outputs.data, 1)\n",
    "    if device == 'cuda':\n",
    "        predicted = predicted.cpu()\n",
    "        \n",
    "    correct = 0\n",
    "    total = 0\n",
    "    total += TestLabels.size(0)\n",
    "    correct += (predicted == TestLabels).sum()\n",
    "    print('At Iteration: %d / %d  ;  Training Loss: %f ; Testing Acc: %f '%(epoch + 1,iterations,runningLoss/\n",
    "                                                                            (TrainImages.shape[0]/\n",
    "                                                                             BatchSize),(100 * correct/ float(total))))\n",
    "print('Finished Training')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Encoder Weights Visualization:\n",
    "======================="
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cll_weights_ft = copy.deepcopy(net[0][0].weight.data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cll_weights = (1 + cll_weights)*127.5\n",
    "cll_weights_ft = (1 + cll_weights_ft)*127.5\n",
    "\n",
    "if use_gpu:\n",
    "    cll_weights = cll_weights.view(-1,280,280).byte().cpu()\n",
    "    cll_weights_ft = cll_weights_ft.view(-1,280,280).byte().cpu()\n",
    "else:\n",
    "    cll_weights = cll_weights.view(-1,280,280).byte()\n",
    "    cll_weights_ft = cll_weights_ft.view(-1,280,280).byte()\n",
    "\n",
    "d_weights = cll_weights - cll_weights_ft\n",
    "\n",
    "fig = plt.figure()\n",
    "plot=fig.add_subplot(1,3,1)\n",
    "img = np.array(cll_weights.numpy())[0]\n",
    "plot.set_title('Encoder Weights')\n",
    "imgplot = plt.imshow(img,cmap='gray')\n",
    "\n",
    "plot=fig.add_subplot(1,3,2)\n",
    "img = np.array(cll_weights_ft.numpy())[0]\n",
    "plot.set_title('Finetuned Weights')\n",
    "imgplot = plt.imshow(img,cmap='gray')\n",
    "\n",
    "plot=fig.add_subplot(1,3,3)\n",
    "img = np.array(d_weights.numpy())[0]\n",
    "plot.set_title('Weight update')\n",
    "imgplot = plt.imshow(img,cmap='gray')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Classifier Weights Visualization:\n",
    "======================="
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "trained_classifier_weights = copy.deepcopy(net.classifier[0].weight.data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "init_classifier_weights = (1 + init_classifier_weights)*255\n",
    "trained_classifier_weights = (1 + trained_classifier_weights)*255\n",
    "\n",
    "if use_gpu:\n",
    "    init_classifier_weights = init_classifier_weights.view(-1,40,25).byte().cpu()\n",
    "    trained_classifier_weights = trained_classifier_weights.view(-1,40,25).byte().cpu()\n",
    "else:\n",
    "    init_classifier_weights = init_classifier_weights.view(-1,40,25).byte()\n",
    "    trained_classifier_weights = trained_classifier_weights.view(-1,40,25).byte()\n",
    "\n",
    "d_weights = init_classifier_weights - trained_classifier_weights\n",
    "\n",
    "fig = plt.figure()\n",
    "plot=fig.add_subplot(1,3,1)\n",
    "img = np.array(init_classifier_weights.numpy())[0]\n",
    "plot.set_title('Initial Weights')\n",
    "imgplot = plt.imshow(img,cmap='gray')\n",
    "\n",
    "plot=fig.add_subplot(1,3,2)\n",
    "img = np.array(trained_classifier_weights.numpy())[0]\n",
    "plot.set_title('Trained Weights')\n",
    "imgplot = plt.imshow(img,cmap='gray')\n",
    "\n",
    "plot=fig.add_subplot(1,3,3)\n",
    "img = np.array(d_weights.numpy())[0]\n",
    "plot.set_title('Weight update')\n",
    "imgplot = plt.imshow(img,cmap='gray')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}