Hill-Valley.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from torch import nn\n",
    "from torch.nn import functional as F\n",
    "from torch import optim\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from train_utils import train_model, BCLogitMetric"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import minmax_scale\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import roc_auc_score, precision_score, accuracy_score, recall_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from classification_uncertainty_utils import aleatoric_uncertainty, epistemic_uncertainty, uncertainty_avg, score_avg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<torch._C.Generator at 0x7fabb5968150>"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.random.seed(1234)\n",
    "torch.manual_seed(1234)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "training_table = pd.read_table(\"./data/hill-valley/Hill_Valley_with_noise_Training.data\", sep=',', dtype=np.float32)\n",
    "testing_table = pd.read_table(\"./data/hill-valley/Hill_Valley_with_noise_Testing.data\", sep=',', dtype=np.float32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "dim = 1\n",
    "seq_length = 100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train = minmax_scale(training_table.drop(\"class\", axis=1).values, axis=1)\n",
    "y_train = training_table[\"class\"].values\n",
    "x_test = minmax_scale(testing_table.drop(\"class\", axis=1).values, axis=1)\n",
    "y_test = testing_table[\"class\"].values\n",
    "\n",
    "x_train = np.expand_dims(x_train, axis=-1)\n",
    "x_test = np.expand_dims(x_test, axis=-1)\n",
    "\n",
    "x_train, x_val, y_train, y_val = train_test_split(x_train, y_train, test_size=0.3, random_state=42)\n",
    "\n",
    "N = len(y_train)\n",
    "\n",
    "train_dl = torch.utils.data.DataLoader(\n",
    "        [(x_train[i], y_train[i]) for i in range(len(y_train))],\n",
    "        batch_size=10,\n",
    "        shuffle=True\n",
    ")\n",
    "val_dl = torch.utils.data.DataLoader(\n",
    "        [(x_val[i], y_val[i]) for i in range(len(y_val))],\n",
    "        batch_size=10,\n",
    "        shuffle=False\n",
    ")\n",
    "test_dl = torch.utils.data.DataLoader(\n",
    "        [(x_test[i], y_test[i]) for i in range(len(y_test))],\n",
    "        batch_size=len(y_test),\n",
    "        shuffle=False\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Net(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(Net, self).__init__()\n",
    "        self.rnn = nn.LSTM(1, 10)\n",
    "        self.fc = nn.Linear(10, 1)\n",
    "\n",
    "    def regularizer(self):        \n",
    "        return 0.\n",
    "\n",
    "    def forward(self, x: torch.Tensor):\n",
    "        out, _ = self.rnn(x)\n",
    "        out = out[-1]\n",
    "        out = torch.relu(out)\n",
    "        out = self.fc(out).flatten()\n",
    "        return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "net = Net()\n",
    "criterion = nn.BCEWithLogitsLoss()\n",
    "\n",
    "optimizer = optim.Adam(net.parameters(), lr=1e-3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "metrics = [BCLogitMetric()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1, 100.00%] train loss: 0.693061\n",
      "[1, 100.00%] validate loss: 0.710546\n",
      "Accuracy = 0.484\n",
      "[2, 100.00%] train loss: 0.665555\n",
      "[2, 100.00%] validate loss: 0.648465\n",
      "Accuracy = 0.742\n",
      "[3, 100.00%] train loss: 0.524788\n",
      "[3, 100.00%] validate loss: 0.428899\n",
      "Accuracy = 0.826\n",
      "[4, 100.00%] train loss: 0.350633\n",
      "[4, 100.00%] validate loss: 0.298503\n",
      "Accuracy = 0.868\n",
      "[5, 100.00%] train loss: 0.249484\n",
      "[5, 100.00%] validate loss: 0.222077\n",
      "Accuracy = 0.893\n",
      "[6, 100.00%] train loss: 0.188720\n",
      "[6, 100.00%] validate loss: 0.162065\n",
      "Accuracy = 0.911\n",
      "[7, 100.00%] train loss: 0.152449\n",
      "[7, 100.00%] validate loss: 0.127921\n",
      "Accuracy = 0.924\n",
      "[8, 100.00%] train loss: 0.127388\n",
      "[8, 100.00%] validate loss: 0.105772\n",
      "Accuracy = 0.933\n",
      "[9, 100.00%] train loss: 0.110659\n",
      "[9, 100.00%] validate loss: 0.090516\n",
      "Accuracy = 0.941\n",
      "[10, 100.00%] train loss: 0.099011\n",
      "[10, 100.00%] validate loss: 0.078570\n",
      "Accuracy = 0.947\n",
      "[11, 100.00%] train loss: 0.089809\n",
      "[11, 100.00%] validate loss: 0.069626\n",
      "Accuracy = 0.952\n",
      "[12, 100.00%] train loss: 0.082904\n",
      "[12, 100.00%] validate loss: 0.062042\n",
      "Accuracy = 0.956\n",
      "[13, 100.00%] train loss: 0.077147\n",
      "[13, 100.00%] validate loss: 0.056292\n",
      "Accuracy = 0.959\n",
      "[14, 100.00%] train loss: 0.072680\n",
      "[14, 100.00%] validate loss: 0.050756\n",
      "Accuracy = 0.962\n",
      "[15, 100.00%] train loss: 0.068613\n",
      "[15, 100.00%] validate loss: 0.046769\n",
      "Accuracy = 0.964\n",
      "[16, 100.00%] train loss: 0.065428\n",
      "[16, 100.00%] validate loss: 0.043379\n",
      "Accuracy = 0.967\n",
      "[17, 100.00%] train loss: 0.055107\n",
      "[17, 100.00%] validate loss: 0.039946\n",
      "Accuracy = 0.969\n",
      "[18, 100.00%] train loss: 0.052427\n",
      "[18, 100.00%] validate loss: 0.036894\n",
      "Accuracy = 0.970\n",
      "[19, 100.00%] train loss: 0.050062\n",
      "[19, 100.00%] validate loss: 0.034302\n",
      "Accuracy = 0.972\n",
      "[20, 100.00%] train loss: 0.040094\n",
      "[20, 100.00%] validate loss: 0.031949\n",
      "Accuracy = 0.973\n",
      "[21, 100.00%] train loss: 0.038012\n",
      "[21, 100.00%] validate loss: 0.029735\n",
      "Accuracy = 0.975\n",
      "[22, 100.00%] train loss: 0.036181\n",
      "[22, 100.00%] validate loss: 0.027673\n",
      "Accuracy = 0.976\n",
      "[23, 100.00%] train loss: 0.034488\n",
      "[23, 100.00%] validate loss: 0.026024\n",
      "Accuracy = 0.977\n",
      "[24, 100.00%] train loss: 0.033039\n",
      "[24, 100.00%] validate loss: 0.024440\n",
      "Accuracy = 0.978\n",
      "[25, 100.00%] train loss: 0.031722\n",
      "[25, 100.00%] validate loss: 0.022999\n",
      "Accuracy = 0.979\n",
      "[26, 100.00%] train loss: 0.030536\n",
      "[26, 100.00%] validate loss: 0.021717\n",
      "Accuracy = 0.980\n",
      "[27, 100.00%] train loss: 0.029462\n",
      "[27, 100.00%] validate loss: 0.020555\n",
      "Accuracy = 0.980\n",
      "[28, 100.00%] train loss: 0.028493\n",
      "[28, 100.00%] validate loss: 0.019479\n",
      "Accuracy = 0.981\n",
      "[29, 100.00%] train loss: 0.027621\n",
      "[29, 100.00%] validate loss: 0.018413\n",
      "Accuracy = 0.982\n",
      "[30, 100.00%] train loss: 0.026778\n",
      "[30, 100.00%] validate loss: 0.017596\n",
      "Accuracy = 0.982\n",
      "[31, 100.00%] train loss: 0.026074\n",
      "[31, 100.00%] validate loss: 0.016641\n",
      "Accuracy = 0.983\n",
      "[32, 100.00%] train loss: 0.025350\n",
      "[32, 100.00%] validate loss: 0.015972\n",
      "Accuracy = 0.983\n",
      "[33, 100.00%] train loss: 0.024696\n",
      "[33, 100.00%] validate loss: 0.015312\n",
      "Accuracy = 0.984\n",
      "[34, 100.00%] train loss: 0.034324\n",
      "[34, 100.00%] validate loss: 0.014650\n",
      "Accuracy = 0.984\n",
      "[35, 100.00%] train loss: 0.033897\n",
      "[35, 100.00%] validate loss: 0.014269\n",
      "Accuracy = 0.985\n",
      "[36, 100.00%] train loss: 0.033540\n",
      "[36, 100.00%] validate loss: 0.013895\n",
      "Accuracy = 0.985\n",
      "[37, 100.00%] train loss: 0.033197\n",
      "[37, 100.00%] validate loss: 0.037590\n",
      "Accuracy = 0.985\n",
      "Retry 1/5\n",
      "[38, 100.00%] train loss: 0.032858\n",
      "[38, 100.00%] validate loss: 0.037251\n",
      "Accuracy = 0.986\n",
      "Retry 2/5\n",
      "[39, 100.00%] train loss: 0.022876\n",
      "[39, 100.00%] validate loss: 0.012439\n",
      "Accuracy = 0.986\n",
      "[40, 100.00%] train loss: 0.032246\n",
      "[40, 100.00%] validate loss: 0.036691\n",
      "Accuracy = 0.986\n",
      "Retry 1/5\n",
      "[41, 100.00%] train loss: 0.022152\n",
      "[41, 100.00%] validate loss: 0.011572\n",
      "Accuracy = 0.987\n",
      "[42, 100.00%] train loss: 0.031701\n",
      "[42, 100.00%] validate loss: 0.036204\n",
      "Accuracy = 0.987\n",
      "Retry 1/5\n",
      "[43, 100.00%] train loss: 0.021499\n",
      "[43, 100.00%] validate loss: 0.010888\n",
      "Accuracy = 0.987\n",
      "[44, 100.00%] train loss: 0.031209\n",
      "[44, 100.00%] validate loss: 0.035792\n",
      "Accuracy = 0.987\n",
      "Retry 1/5\n",
      "[45, 100.00%] train loss: 0.020966\n",
      "[45, 100.00%] validate loss: 0.035476\n",
      "Accuracy = 0.987\n",
      "Retry 2/5\n",
      "[46, 100.00%] train loss: 0.020622\n",
      "[46, 100.00%] validate loss: 0.009952\n",
      "Accuracy = 0.988\n",
      "[47, 100.00%] train loss: 0.030627\n",
      "[47, 100.00%] validate loss: 0.035194\n",
      "Accuracy = 0.988\n",
      "Retry 1/5\n",
      "[48, 100.00%] train loss: 0.020264\n",
      "[48, 100.00%] validate loss: 0.034983\n",
      "Accuracy = 0.988\n",
      "Retry 2/5\n",
      "[49, 100.00%] train loss: 0.019935\n",
      "[49, 100.00%] validate loss: 0.009149\n",
      "Accuracy = 0.988\n",
      "[50, 100.00%] train loss: 0.030012\n",
      "[50, 100.00%] validate loss: 0.034789\n",
      "Accuracy = 0.988\n",
      "Retry 1/5\n",
      "[51, 100.00%] train loss: 0.019648\n",
      "[51, 100.00%] validate loss: 0.034557\n",
      "Accuracy = 0.988\n",
      "Retry 2/5\n",
      "[52, 100.00%] train loss: 0.019385\n",
      "[52, 100.00%] validate loss: 0.008479\n",
      "Accuracy = 0.989\n",
      "[53, 100.00%] train loss: 0.019160\n",
      "[53, 100.00%] validate loss: 0.008236\n",
      "Accuracy = 0.989\n",
      "[54, 100.00%] train loss: 0.029422\n",
      "[54, 100.00%] validate loss: 0.008029\n",
      "Accuracy = 0.989\n",
      "[55, 100.00%] train loss: 0.018954\n",
      "[55, 100.00%] validate loss: 0.034135\n",
      "Accuracy = 0.989\n",
      "Retry 1/5\n",
      "[56, 100.00%] train loss: 0.018783\n",
      "[56, 100.00%] validate loss: 0.033998\n",
      "Accuracy = 0.989\n",
      "Retry 2/5\n",
      "[57, 100.00%] train loss: 0.018561\n",
      "[57, 100.00%] validate loss: 0.033843\n",
      "Accuracy = 0.989\n",
      "Retry 3/5\n",
      "[58, 100.00%] train loss: 0.018412\n",
      "[58, 100.00%] validate loss: 0.007320\n",
      "Accuracy = 0.990\n",
      "[59, 100.00%] train loss: 0.028727\n",
      "[59, 100.00%] validate loss: 0.033808\n",
      "Accuracy = 0.990\n",
      "Retry 1/5\n",
      "[60, 100.00%] train loss: 0.018266\n",
      "[60, 100.00%] validate loss: 0.033636\n",
      "Accuracy = 0.990\n",
      "Retry 2/5\n",
      "[61, 100.00%] train loss: 0.018083\n",
      "[61, 100.00%] validate loss: 0.033521\n",
      "Accuracy = 0.990\n",
      "Retry 3/5\n",
      "[62, 100.00%] train loss: 0.017939\n",
      "[62, 100.00%] validate loss: 0.033431\n",
      "Accuracy = 0.990\n",
      "Retry 4/5\n",
      "[63, 100.00%] train loss: 0.017813\n",
      "[63, 100.00%] validate loss: 0.006580\n",
      "Accuracy = 0.990\n",
      "[64, 100.00%] train loss: 0.027914\n",
      "[64, 100.00%] validate loss: 0.033278\n",
      "Accuracy = 0.990\n",
      "Retry 1/5\n",
      "[65, 100.00%] train loss: 0.017738\n",
      "[65, 100.00%] validate loss: 0.033303\n",
      "Accuracy = 0.990\n",
      "Retry 2/5\n",
      "[66, 100.00%] train loss: 0.017585\n",
      "[66, 100.00%] validate loss: 0.033148\n",
      "Accuracy = 0.990\n",
      "Retry 3/5\n",
      "[67, 100.00%] train loss: 0.017456\n",
      "[67, 100.00%] validate loss: 0.033045\n",
      "Accuracy = 0.990\n",
      "Retry 4/5\n",
      "[68, 100.00%] train loss: 0.017299\n",
      "[68, 100.00%] validate loss: 0.032935\n",
      "Accuracy = 0.990\n",
      "Retry 5/5\n",
      "CPU times: user 2min 12s, sys: 6.29 s, total: 2min 19s\n",
      "Wall time: 2min 6s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "net = train_model(net, train_dl, val_dl, criterion, optimizer, \"hill-valley.pt\", 100, metrics=metrics, patience=5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy = 1.00\n",
      "Precision = 1.00\n",
      "AUCROC = 1.00\n"
     ]
    }
   ],
   "source": [
    "net.eval()\n",
    "with torch.no_grad():\n",
    "    for i, (x, y) in enumerate(test_dl):\n",
    "        pre_sigmoids = net(x.transpose(-2,-3))\n",
    "        scores = torch.sigmoid(pre_sigmoids)\n",
    "        predictions = torch.round(scores)\n",
    "        print(\"Accuracy = %.2f\" % accuracy_score(y, predictions))\n",
    "        print(\"Precision = %.2f\" % precision_score(y, predictions))\n",
    "        print(\"AUCROC = %.2f\" % roc_auc_score(y, scores))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Flat line test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_x_test = minmax_scale(np.full((2, 100), 0., dtype=np.float32), axis=1)\n",
    "new_x_test = np.expand_dims(new_x_test, axis=-1)\n",
    "new_x_test = new_x_test.swapaxes(-2,-3)\n",
    "new_x_test = torch.tensor(new_x_test)\n",
    "\n",
    "net.eval()\n",
    "with torch.no_grad():\n",
    "    outputs = net(new_x_test)\n",
    "    outputs = torch.sigmoid(outputs)\n",
    "    scores = outputs\n",
    "\n",
    "plt.plot(new_x_test[:,0])\n",
    "plt.show()\n",
    "print(\"predictions\", scores)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Noise test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_x_test = minmax_scale(np.random.randn(2, 100).astype(np.float32), axis=1)\n",
    "new_x_test = np.expand_dims(new_x_test, axis=-1)\n",
    "new_x_test = new_x_test.swapaxes(-2,-3)\n",
    "new_x_test = torch.tensor(new_x_test)\n",
    "\n",
    "net.eval()\n",
    "with torch.no_grad():\n",
    "    outputs = net(new_x_test)\n",
    "    outputs = torch.sigmoid(outputs)\n",
    "    scores = outputs\n",
    "\n",
    "plt.plot(new_x_test[:,0])\n",
    "plt.show()\n",
    "print(\"predictions\", scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}