lecture39.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Lecture 39: ResNet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import copy\n",
    "import time\n",
    "import tqdm\n",
    "import torch\n",
    "import numpy as np\n",
    "import torchvision\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import torch.nn.functional as F\n",
    "import matplotlib.pyplot as plt\n",
    "from torchvision import transforms,datasets, models\n",
    "\n",
    "print(torch.__version__) # This code has been updated for PyTorch 1.0.0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Data:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "apply_transform = transforms.Compose([transforms.Resize(224),transforms.ToTensor()])\n",
    "BatchSize = 1\n",
    "\n",
    "trainset = datasets.CIFAR10(root='./CIFAR10', train=True, download=True, transform=apply_transform)\n",
    "trainLoader = torch.utils.data.DataLoader(trainset, batch_size=BatchSize,\n",
    "                                          shuffle=True, num_workers=4) # Creating dataloader\n",
    "\n",
    "testset = datasets.CIFAR10(root='./CIFAR10', train=False, download=True, transform=apply_transform)\n",
    "testLoader = torch.utils.data.DataLoader(testset, batch_size=BatchSize,\n",
    "                                         shuffle=False, num_workers=4) # Creating dataloader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Size of train and test datasets\n",
    "print('No. of samples in train set: '+str(len(trainLoader.dataset)))\n",
    "print('No. of samples in test set: '+str(len(testLoader.dataset)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define network architecture"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "net = models.resnet18()\n",
    "print(net)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Counting number of trainable parameters\n",
    "totalParams = 0\n",
    "for name,params in net.named_parameters():\n",
    "    print(name,'-->',params.size())\n",
    "    totalParams += np.sum(np.prod(params.size()))\n",
    "print('Total number of parameters: '+str(totalParams))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Modifying the last fully-connected layer for 10 classes\n",
    "net.fc = nn.Linear(512,10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Copying initial weights for visualization\n",
    "init_weightConv1 = copy.deepcopy(net.conv1.weight.data) # 1st conv layer\n",
    "init_weightConv2 = copy.deepcopy(net.layer1[0].conv1.weight.data) # 2nd conv layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Check availability of GPU\n",
    "\n",
    "use_gpu = torch.cuda.is_available()\n",
    "# use_gpu = False # Uncomment in case of GPU memory error\n",
    "if use_gpu:\n",
    "    print('GPU is available!')\n",
    "    device = \"cuda\"\n",
    "else:\n",
    "    print('GPU is not available!')\n",
    "    device = \"cpu\"\n",
    "    \n",
    "net = net.to(device)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define loss function and optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "criterion = nn.NLLLoss() # Negative Log-likelihood\n",
    "optimizer = optim.Adam(net.parameters(), lr=1e-4) # Adam"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train the network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "iterations = 10\n",
    "trainLoss = []\n",
    "trainAuxLoss = []\n",
    "trainTotalLoss = []\n",
    "trainAcc = []\n",
    "\n",
    "testLoss = []\n",
    "testAcc = []\n",
    "\n",
    "start = time.time()\n",
    "for epoch in range(iterations):\n",
    "    epochStart = time.time()\n",
    "    runningLoss = 0.0   \n",
    "    avgTotalLoss = 0.0\n",
    "    running_correct = 0\n",
    "    net.train() # For training\n",
    "    for data in tqdm.tqdm_notebook(trainLoader):\n",
    "        # Initialize gradients to zero\n",
    "        optimizer.zero_grad()\n",
    "        \n",
    "        inputs,labels = data        \n",
    "        inputs, labels = inputs.to(device), labels.to(device)  \n",
    "        # Feed-forward input data through the network        \n",
    "        outputs = net(inputs) \n",
    "        _, predicted = torch.max(outputs.data, 1)\n",
    "        running_correct += (predicted == labels.data).sum()          \n",
    "               \n",
    "        # Compute loss/error\n",
    "        loss = criterion(F.log_softmax(outputs,dim=1), labels)\n",
    "        # Backpropagate loss and compute gradients\n",
    "        loss.backward()\n",
    "        # Update the network parameters\n",
    "        optimizer.step()\n",
    "        # Accumulate loss per batch\n",
    "        runningLoss += loss.item()  \n",
    "              \n",
    "    avgTrainAcc = 100*float(running_correct)/50000.0\n",
    "    avgTrainLoss = runningLoss/(50000.0  /BatchSize)  \n",
    "    trainAcc.append(avgTrainAcc)\n",
    "    trainLoss.append(avgTrainLoss)    \n",
    "    \n",
    "    # Evaluating performance on test set for each epoch\n",
    "    net.eval() # For testing [Affects batch-norm and dropout layers (if any)]\n",
    "    running_correct = 0\n",
    "    with torch.no_grad():\n",
    "        for data in testLoader:\n",
    "            inputs,labels = data            \n",
    "            inputs, labels = inputs.to(device), labels.to(device)\n",
    "            outputs = net(inputs)\n",
    "            _, predicted = torch.max(outputs.data, 1)\n",
    "            running_correct += (predicted == labels.data).sum()\n",
    "\n",
    "            loss = criterion(F.log_softmax(outputs,dim=1), labels)\n",
    "            runningLoss += loss.item()  \n",
    "            \n",
    "            \n",
    "            \n",
    "    avgTestLoss = runningLoss/(10000.0/BatchSize)\n",
    "    avgTestAcc = 100*float(running_correct)/10000.0\n",
    "    testLoss.append(avgTestLoss)\n",
    "    testAcc.append(avgTestAcc)\n",
    "        \n",
    "    \n",
    "    # Plotting training loss vs Epochs\n",
    "    fig1 = plt.figure(1)        \n",
    "    plt.plot(range(epoch+1),trainLoss,'r-',label='train')  \n",
    "    plt.plot(range(epoch+1),testLoss,'g-',label='test') \n",
    "    if epoch==0:\n",
    "        plt.legend(loc='upper left')\n",
    "        plt.xlabel('Epochs')\n",
    "        plt.ylabel('Loss')   \n",
    "    # Plotting testing accuracy vs Epochs\n",
    "    fig2 = plt.figure(2)        \n",
    "    plt.plot(range(epoch+1),trainAcc,'r-',label='train')    \n",
    "    plt.plot(range(epoch+1),testAcc,'g-',label='test')        \n",
    "    if epoch==0:\n",
    "        plt.legend(loc='upper left')\n",
    "        plt.xlabel('Epochs')\n",
    "        plt.ylabel('Accuracy')    \n",
    "    epochEnd = time.time()-epochStart\n",
    "    print('Iteration: {:.0f} /{:.0f}  ;  Training Loss: {:.6f} ; Testing Acc: {:.3f} ; Time consumed: {:.0f}m {:.0f}s '\\\n",
    "          .format(epoch + 1,iterations,avgTrainLoss,avgTestAcc,epochEnd//60,epochEnd%60))\n",
    "end = time.time()-start\n",
    "print('Training completed in {:.0f}m {:.0f}s'.format(end//60,end%60))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Copying trained weights for visualization\n",
    "trained_weightConv1 = copy.deepcopy(net.conv1.weight.data)\n",
    "trained_weightConv2 = copy.deepcopy(net.layer1[0].conv1.weight.data)\n",
    "if use_gpu:\n",
    "    trained_weightConv1 = trained_weightConv1.cpu()\n",
    "    trained_weightConv2 = trained_weightConv2.cpu()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## Visualization of weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# functions to show an image\n",
    "def imshow(img, strlabel):\n",
    "    npimg = img.numpy()\n",
    "    npimg = np.abs(npimg)\n",
    "    fig_size = plt.rcParams[\"figure.figsize\"]\n",
    "    fig_size[0] = 10\n",
    "    fig_size[1] = 10\n",
    "    plt.rcParams[\"figure.figsize\"] = fig_size\n",
    "    plt.figure()\n",
    "    plt.title(strlabel)\n",
    "    plt.imshow(np.transpose(npimg, (1, 2, 0)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Visualizing weights of 1st convolutional layer\n",
    "imshow(torchvision.utils.make_grid(init_weightConv1,nrow=8,normalize=True),'Initial weights: conv1')\n",
    "imshow(torchvision.utils.make_grid(trained_weightConv1,nrow=8,normalize=True),'Trained weights: conv1')\n",
    "imshow(torchvision.utils.make_grid(init_weightConv1-trained_weightConv1,nrow=8,normalize=True),'Difference of weights: conv1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Visualizing weights of 2nd convolutional layer\n",
    "imshow(torchvision.utils.make_grid(init_weightConv2[0].unsqueeze(1),nrow=8,normalize=True),'Initial weights: conv2')\n",
    "imshow(torchvision.utils.make_grid(trained_weightConv2[0].unsqueeze(1),nrow=8,normalize=True),'Trained weights: conv2')\n",
    "imshow(torchvision.utils.make_grid(init_weightConv2[0].unsqueeze(1)-trained_weightConv2[0].unsqueeze(1),nrow=8,normalize=True),'Difference of weights: conv2')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}