Bitcoin Ransomware Detection Dataset.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Bitcoin Ransomware Dataset\n",
    "See https://archive.ics.uci.edu/ml/datasets/BitcoinHeistRansomwareAddressDataset for dataset information and feature descriptions."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "%config InlineBackend.figure_format = 'retina'\n",
    "\n",
    "import csv\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import pandas_profiling\n",
    "import matplotlib.pyplot as plt\n",
    "from scipy import stats\n",
    "import pickle\n",
    "import operator\n",
    "import glob\n",
    "from scipy.io.arff import loadarff \n",
    "from scipy.stats import ttest_rel\n",
    "\n",
    "import seaborn as sns; sns.set_style('white')\n",
    "\n",
    "from sklearn.utils import resample\n",
    "from sklearn.metrics import accuracy_score, plot_confusion_matrix, f1_score, plot_roc_curve, roc_auc_score, make_scorer\n",
    "from sklearn.model_selection import KFold, GridSearchCV, RandomizedSearchCV\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.svm import SVC, LinearSVC\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.ensemble import AdaBoostClassifier\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "\n",
    "from sklearn.model_selection import cross_val_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>address</th>\n",
       "      <th>year</th>\n",
       "      <th>day</th>\n",
       "      <th>length</th>\n",
       "      <th>weight</th>\n",
       "      <th>cnt</th>\n",
       "      <th>looped</th>\n",
       "      <th>neighbors</th>\n",
       "      <th>income</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>111K8kZAEnJg245r2cM6y9zgJGHZtJPy6</td>\n",
       "      <td>2017</td>\n",
       "      <td>11</td>\n",
       "      <td>18</td>\n",
       "      <td>0.008333</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1.000500e+08</td>\n",
       "      <td>princetonCerber</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1123pJv8jzeFQaCV4w644pzQJzVWay2zcA</td>\n",
       "      <td>2016</td>\n",
       "      <td>132</td>\n",
       "      <td>44</td>\n",
       "      <td>0.000244</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1.000000e+08</td>\n",
       "      <td>princetonLocky</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>112536im7hy6wtKbpH1qYDWtTyMRAcA2p7</td>\n",
       "      <td>2016</td>\n",
       "      <td>246</td>\n",
       "      <td>0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>2.000000e+08</td>\n",
       "      <td>princetonCerber</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1126eDRw2wqSkWosjTCre8cjjQW8sSeWH7</td>\n",
       "      <td>2016</td>\n",
       "      <td>322</td>\n",
       "      <td>72</td>\n",
       "      <td>0.003906</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>7.120000e+07</td>\n",
       "      <td>princetonCerber</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1129TSjKtx65E35GiUo4AYVeyo48twbrGX</td>\n",
       "      <td>2016</td>\n",
       "      <td>238</td>\n",
       "      <td>144</td>\n",
       "      <td>0.072848</td>\n",
       "      <td>456</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2.000000e+08</td>\n",
       "      <td>princetonLocky</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2916692</th>\n",
       "      <td>12D3trgho1vJ4mGtWBRPyHdMJK96TRYSry</td>\n",
       "      <td>2018</td>\n",
       "      <td>330</td>\n",
       "      <td>0</td>\n",
       "      <td>0.111111</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1.255809e+09</td>\n",
       "      <td>white</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2916693</th>\n",
       "      <td>1P7PputTcVkhXBmXBvSD9MJ3UYPsiou1u2</td>\n",
       "      <td>2018</td>\n",
       "      <td>330</td>\n",
       "      <td>0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>4.409699e+07</td>\n",
       "      <td>white</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2916694</th>\n",
       "      <td>1KYiKJEfdJtap9QX2v9BXJMpz2SfU4pgZw</td>\n",
       "      <td>2018</td>\n",
       "      <td>330</td>\n",
       "      <td>2</td>\n",
       "      <td>12.000000</td>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "      <td>35</td>\n",
       "      <td>2.398267e+09</td>\n",
       "      <td>white</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2916695</th>\n",
       "      <td>15iPUJsRNZQZHmZZVwmQ63srsmughCXV4a</td>\n",
       "      <td>2018</td>\n",
       "      <td>330</td>\n",
       "      <td>0</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1.780427e+08</td>\n",
       "      <td>white</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2916696</th>\n",
       "      <td>3LFFBxp15h9KSFtaw55np8eP5fv6kdK17e</td>\n",
       "      <td>2018</td>\n",
       "      <td>330</td>\n",
       "      <td>144</td>\n",
       "      <td>0.073972</td>\n",
       "      <td>6800</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1.123500e+08</td>\n",
       "      <td>white</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2916697 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                    address  year  day  length     weight  \\\n",
       "0         111K8kZAEnJg245r2cM6y9zgJGHZtJPy6  2017   11      18   0.008333   \n",
       "1        1123pJv8jzeFQaCV4w644pzQJzVWay2zcA  2016  132      44   0.000244   \n",
       "2        112536im7hy6wtKbpH1qYDWtTyMRAcA2p7  2016  246       0   1.000000   \n",
       "3        1126eDRw2wqSkWosjTCre8cjjQW8sSeWH7  2016  322      72   0.003906   \n",
       "4        1129TSjKtx65E35GiUo4AYVeyo48twbrGX  2016  238     144   0.072848   \n",
       "...                                     ...   ...  ...     ...        ...   \n",
       "2916692  12D3trgho1vJ4mGtWBRPyHdMJK96TRYSry  2018  330       0   0.111111   \n",
       "2916693  1P7PputTcVkhXBmXBvSD9MJ3UYPsiou1u2  2018  330       0   1.000000   \n",
       "2916694  1KYiKJEfdJtap9QX2v9BXJMpz2SfU4pgZw  2018  330       2  12.000000   \n",
       "2916695  15iPUJsRNZQZHmZZVwmQ63srsmughCXV4a  2018  330       0   0.500000   \n",
       "2916696  3LFFBxp15h9KSFtaw55np8eP5fv6kdK17e  2018  330     144   0.073972   \n",
       "\n",
       "          cnt  looped  neighbors        income            label  \n",
       "0           1       0          2  1.000500e+08  princetonCerber  \n",
       "1           1       0          1  1.000000e+08   princetonLocky  \n",
       "2           1       0          2  2.000000e+08  princetonCerber  \n",
       "3           1       0          2  7.120000e+07  princetonCerber  \n",
       "4         456       0          1  2.000000e+08   princetonLocky  \n",
       "...       ...     ...        ...           ...              ...  \n",
       "2916692     1       0          1  1.255809e+09            white  \n",
       "2916693     1       0          1  4.409699e+07            white  \n",
       "2916694     6       6         35  2.398267e+09            white  \n",
       "2916695     1       0          1  1.780427e+08            white  \n",
       "2916696  6800       0          2  1.123500e+08            white  \n",
       "\n",
       "[2916697 rows x 10 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv('BitCoinHeistData.csv')\n",
    "df = df.rename(columns={'count':'cnt'})\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>length</th>\n",
       "      <th>weight</th>\n",
       "      <th>cnt</th>\n",
       "      <th>looped</th>\n",
       "      <th>neighbors</th>\n",
       "      <th>income</th>\n",
       "      <th>new_label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>18</td>\n",
       "      <td>0.008333</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1.000500e+08</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>44</td>\n",
       "      <td>0.000244</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1.000000e+08</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>2.000000e+08</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>72</td>\n",
       "      <td>0.003906</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>7.120000e+07</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>144</td>\n",
       "      <td>0.072848</td>\n",
       "      <td>456</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2.000000e+08</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2916692</th>\n",
       "      <td>0</td>\n",
       "      <td>0.111111</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1.255809e+09</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2916693</th>\n",
       "      <td>0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>4.409699e+07</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2916694</th>\n",
       "      <td>2</td>\n",
       "      <td>12.000000</td>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "      <td>35</td>\n",
       "      <td>2.398267e+09</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2916695</th>\n",
       "      <td>0</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1.780427e+08</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2916696</th>\n",
       "      <td>144</td>\n",
       "      <td>0.073972</td>\n",
       "      <td>6800</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1.123500e+08</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2916697 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         length     weight   cnt  looped  neighbors        income  new_label\n",
       "0            18   0.008333     1       0          2  1.000500e+08          1\n",
       "1            44   0.000244     1       0          1  1.000000e+08          1\n",
       "2             0   1.000000     1       0          2  2.000000e+08          1\n",
       "3            72   0.003906     1       0          2  7.120000e+07          1\n",
       "4           144   0.072848   456       0          1  2.000000e+08          1\n",
       "...         ...        ...   ...     ...        ...           ...        ...\n",
       "2916692       0   0.111111     1       0          1  1.255809e+09          0\n",
       "2916693       0   1.000000     1       0          1  4.409699e+07          0\n",
       "2916694       2  12.000000     6       6         35  2.398267e+09          0\n",
       "2916695       0   0.500000     1       0          1  1.780427e+08          0\n",
       "2916696     144   0.073972  6800       0          2  1.123500e+08          0\n",
       "\n",
       "[2916697 rows x 7 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 1 = ransomware; 0 = whitelist\n",
    "df['new_label'] = np.where(df['label']=='white', 0, 1)\n",
    "df = df.drop(columns=['year', 'day', 'address', 'label'])\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    2875284\n",
       "1      41413\n",
       "Name: new_label, dtype: int64"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['new_label'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1    6000\n",
       "0    6000\n",
       "Name: new_label, dtype: int64"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Separate majority and minority classes\n",
    "df_majority = df[df.new_label==0]\n",
    "df_minority = df[df.new_label==1]\n",
    "\n",
    "# Downsample majority and minority class\n",
    "df_majority_downsampled = resample(df_majority, \n",
    "                                 replace=False,    # sample without replacement\n",
    "                                 n_samples=6000,     # to match minority class\n",
    "                                 random_state=123) # reproducible results\n",
    "\n",
    "df_minority_downsampled = resample(df_minority, \n",
    "                                 replace=False,    # sample without replacement\n",
    "                                 n_samples=6000,     # to match majority class\n",
    "                                 random_state=123) # reproducible results\n",
    " \n",
    "# Combine minority class with downsampled majority class\n",
    "df_downsampled = pd.concat([df_majority_downsampled, df_minority_downsampled])\n",
    " \n",
    "# Display new class counts\n",
    "df_downsampled.new_label.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>length</th>\n",
       "      <th>weight</th>\n",
       "      <th>cnt</th>\n",
       "      <th>looped</th>\n",
       "      <th>neighbors</th>\n",
       "      <th>income</th>\n",
       "      <th>new_label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2488416</th>\n",
       "      <td>2</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>6.089024e+08</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2873770</th>\n",
       "      <td>2</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>3.969468e+08</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2537321</th>\n",
       "      <td>50</td>\n",
       "      <td>0.022457</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1.228130e+08</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2868318</th>\n",
       "      <td>144</td>\n",
       "      <td>0.077579</td>\n",
       "      <td>4825</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>2.000000e+08</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>636908</th>\n",
       "      <td>0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>5.605605e+09</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23778</th>\n",
       "      <td>2</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>3.000000e+08</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6969</th>\n",
       "      <td>0</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1.220000e+08</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36643</th>\n",
       "      <td>0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>5.000000e+07</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31367</th>\n",
       "      <td>4</td>\n",
       "      <td>1.588816</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>8</td>\n",
       "      <td>8.068000e+08</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3945</th>\n",
       "      <td>0</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1.200000e+08</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>12000 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         length    weight   cnt  looped  neighbors        income  new_label\n",
       "2488416       2  1.000000     1       0          2  6.089024e+08          0\n",
       "2873770       2  0.500000     1       0          2  3.969468e+08          0\n",
       "2537321      50  0.022457     6       0          2  1.228130e+08          0\n",
       "2868318     144  0.077579  4825       0          2  2.000000e+08          0\n",
       "636908        0  1.000000     1       0          2  5.605605e+09          0\n",
       "...         ...       ...   ...     ...        ...           ...        ...\n",
       "23778         2  0.500000     1       0          1  3.000000e+08          1\n",
       "6969          0  0.500000     1       0          2  1.220000e+08          1\n",
       "36643         0  1.000000     1       0          1  5.000000e+07          1\n",
       "31367         4  1.588816     4       0          8  8.068000e+08          1\n",
       "3945          0  0.500000     1       0          2  1.200000e+08          1\n",
       "\n",
       "[12000 rows x 7 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_downsampled"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "df = df_downsampled.copy()\n",
    "scaler = StandardScaler()\n",
    "X, y = df.iloc[:,0:6].to_numpy(), df.iloc[:,6:].to_numpy()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Hyperparameter Search & Experimentation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def experiment():\n",
    "    pipeline1 = Pipeline((\n",
    "    ('clf', RandomForestClassifier()),\n",
    "    ))\n",
    "\n",
    "    pipeline2 = Pipeline((\n",
    "    ('clf', KNeighborsClassifier()),\n",
    "    ))\n",
    "\n",
    "    pipeline3 = Pipeline((\n",
    "    ('clf', AdaBoostClassifier()),\n",
    "    ))\n",
    "    \n",
    "    pipeline4 = Pipeline((\n",
    "    ('clf', LogisticRegression()),\n",
    "    ))\n",
    "    \n",
    "    pipeline5 = Pipeline((\n",
    "    ('clf', MLPClassifier()),\n",
    "    ))\n",
    "    \n",
    "    # Random Forest\n",
    "    parameters1 = {\n",
    "    'clf__n_estimators': [1024],\n",
    "    'clf__max_features': [1, 2, 4, 6, 8, 12, 16, 20]\n",
    "    }\n",
    "\n",
    "    # KNN\n",
    "    parameters2 = {\n",
    "    'clf__n_neighbors': [1,5,9,13,17,21,25,29,33,37,41,45,49,53,57,61,65,69,73,77,81,85,89,93,97,101,105],\n",
    "    'clf__weights': ['uniform', 'distance']\n",
    "    }\n",
    "    \n",
    "    # AdaBoost (Boosted Decision Tree)\n",
    "    parameters3 = {\n",
    "        'clf__algorithm': ['SAMME.R'],\n",
    "        'clf__n_estimators': [2,4,8,16,32,64,128,256,512,1024,2048],\n",
    "        'clf__learning_rate': [1e-3, 1e-2, 1e-1, 1e0, 1e1, 2e1, 5e1]\n",
    "    }\n",
    "    \n",
    "    # Logistic\n",
    "    parameters4 = {\n",
    "    'clf__penalty':['l1', 'l2', None],\n",
    "    'clf__C':[1e-8, 1e-7, 1e-6, 1e-5, 1e-4, 1e-3, 1e-2, 1e-1, 0, 1e0, 1e1, 1e2, 1e3, 1e4],\n",
    "    'clf__max_iter':[5000]\n",
    "    }\n",
    "    \n",
    "    # Multi-layer Perceptron\n",
    "    parameters5 = {\n",
    "        'clf__hidden_layer_sizes':[(1,), (2,), (4,), (8,), (32,), (128,)],\n",
    "        'clf__solver':['sgd'],\n",
    "        'clf__activation':['relu'], \n",
    "        'clf__learning_rate':['constant', 'invscaling'], \n",
    "        'clf__learning_rate_init': [1e-3, 1e-2, 1e-1, 1e0],\n",
    "        'clf__max_iter': [2, 4, 8, 16, 32, 64, 128, 256, 512]\n",
    "    }\n",
    "    \n",
    "    pars = [parameters1, parameters2, parameters3, parameters4, parameters5]\n",
    "    pips = [pipeline1, pipeline2, pipeline3, pipeline4, pipeline5]\n",
    "    \n",
    "    # List of dictionaries to hold the scores of the various metrics for each type of classifier\n",
    "    best_clf_list = []\n",
    "    trial_storage = {}\n",
    "    training_storage = {}\n",
    "    \n",
    "    print(\"starting Gridsearch\")\n",
    "    for i in range(len(pars)):\n",
    "        trial_averages = []\n",
    "        train_performance = []\n",
    "        for t in range(5):\n",
    "            # split and scale data\n",
    "            X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=1/6, random_state=t)\n",
    "            X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size=0.5, random_state=t)\n",
    "            X_train = scaler.fit_transform(X_train)\n",
    "            X_val = scaler.transform(X_val)\n",
    "            X_test = scaler.transform(X_test)\n",
    "                \n",
    "            clf = GridSearchCV(pips[i], pars[i], refit=False, n_jobs=8, cv=5, verbose=3, scoring=('accuracy', 'roc_auc', 'f1'))\n",
    "            clf = clf.fit(X_val, y_val)\n",
    "            \n",
    "            print(\"finished Gridsearch trial \" + str(t + 1) + \" classifier \" + str(i + 1))\n",
    "            print(\"\")\n",
    "            print(\"\")\n",
    "            \n",
    "            # find the best params for each metric in a given trial \n",
    "            best_index_acc = np.argmin(clf.cv_results_['rank_test_accuracy'])\n",
    "            best_params_acc = clf.cv_results_['params'][best_index_acc]\n",
    "            best_index_roc = np.argmin(clf.cv_results_['rank_test_roc_auc'])\n",
    "            best_params_roc = clf.cv_results_['params'][best_index_roc]\n",
    "            best_index_f1 = np.argmin(clf.cv_results_['rank_test_f1'])\n",
    "            best_params_f1 = clf.cv_results_['params'][best_index_f1]\n",
    "    \n",
    "            # train and test models for given metric with their corresponding best parameter settings\n",
    "            pipe = pips[i]\n",
    "            clf_acc = pipe.set_params(**best_params_acc)\n",
    "            clf_acc = clf_acc.fit(X_train, y_train)\n",
    "            clf_roc = pipe.set_params(**best_params_roc)\n",
    "            clf_roc = clf_roc.fit(X_train, y_train)\n",
    "            clf_f1 = pipe.set_params(**best_params_f1)\n",
    "            clf_f1 = clf_f1.fit(X_train, y_train)\n",
    "            \n",
    "            # get training set performance\n",
    "            train_acc = accuracy_score(y_train, clf_acc.predict(X_train))\n",
    "            train_roc = roc_auc_score(y_train, clf_roc.predict_proba(X_train)[:, 1])\n",
    "            train_f1 = f1_score(y_train, clf_f1.predict(X_train))\n",
    "            \n",
    "            train_performance.append({\n",
    "                'Model #': i + 1,\n",
    "                'average': (train_f1 + train_acc + train_roc)/3,\n",
    "                'accuracy': train_acc,\n",
    "                'roc_auc_score': train_roc,\n",
    "                'f1 score': train_f1\n",
    "            })\n",
    "            \n",
    "            # get test set performances \n",
    "            trial_acc = clf_acc.score(X_test, y_test)\n",
    "            trial_roc = roc_auc_score(y_test, clf_roc.predict_proba(X_test)[:, 1])\n",
    "            trial_f1 = f1_score(y_test, clf_f1.predict(X_test))\n",
    "            \n",
    "            # store scores and their averages in list containing averages for each trial\n",
    "            trial_averages.append({\n",
    "                'Model #': i + 1, # model number corresponds to the numbers used in pipeline above (i.e. 1 = Random Forest)\n",
    "                'average':(trial_acc + trial_roc + trial_f1) / 3,\n",
    "                'accuracy': trial_acc,\n",
    "                'roc_auc_score': trial_roc,\n",
    "                'f1_score': trial_f1\n",
    "            })\n",
    "            \n",
    "            train_performance.append({\n",
    "                'Model #': i + 1, # model number corresponds to the numbers used in pipeline above (i.e. 1 = Random Forest)\n",
    "                'average':(train_acc + train_roc + train_f1) / 3,\n",
    "                'accuracy': train_acc,\n",
    "                'roc_auc_score': train_roc,\n",
    "                'f1_score': train_f1\n",
    "            })\n",
    "            \n",
    "        # find the trial with the best average metric scores and append those scores as a dict to best clf list\n",
    "        max_average = 0\n",
    "        for trial in trial_averages:\n",
    "            if trial['average'] > max_average:\n",
    "                max_average = trial['average']\n",
    "                best_trial = trial\n",
    "\n",
    "        best_clf_list.append(best_trial)\n",
    "        training_storage[str(i + 1)]=train_performance\n",
    "        trial_storage[str(i + 1)]=trial_averages\n",
    "    \n",
    "    return best_clf_list, trial_storage, training_storage"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "starting Gridsearch\n",
      "Fitting 5 folds for each of 8 candidates, totalling 40 fits\n",
      "finished Gridsearch trial 1 classifier 1\n",
      "\n",
      "\n",
      "Fitting 5 folds for each of 8 candidates, totalling 40 fits\n",
      "finished Gridsearch trial 2 classifier 1\n",
      "\n",
      "\n",
      "Fitting 5 folds for each of 8 candidates, totalling 40 fits\n",
      "finished Gridsearch trial 3 classifier 1\n",
      "\n",
      "\n",
      "Fitting 5 folds for each of 8 candidates, totalling 40 fits\n",
      "finished Gridsearch trial 4 classifier 1\n",
      "\n",
      "\n",
      "Fitting 5 folds for each of 8 candidates, totalling 40 fits\n",
      "finished Gridsearch trial 5 classifier 1\n",
      "\n",
      "\n",
      "Fitting 5 folds for each of 54 candidates, totalling 270 fits\n",
      "finished Gridsearch trial 1 classifier 2\n",
      "\n",
      "\n",
      "Fitting 5 folds for each of 54 candidates, totalling 270 fits\n",
      "finished Gridsearch trial 2 classifier 2\n",
      "\n",
      "\n",
      "Fitting 5 folds for each of 54 candidates, totalling 270 fits\n",
      "finished Gridsearch trial 3 classifier 2\n",
      "\n",
      "\n",
      "Fitting 5 folds for each of 54 candidates, totalling 270 fits\n",
      "finished Gridsearch trial 4 classifier 2\n",
      "\n",
      "\n",
      "Fitting 5 folds for each of 54 candidates, totalling 270 fits\n",
      "finished Gridsearch trial 5 classifier 2\n",
      "\n",
      "\n",
      "Fitting 5 folds for each of 77 candidates, totalling 385 fits\n",
      "finished Gridsearch trial 1 classifier 3\n",
      "\n",
      "\n",
      "Fitting 5 folds for each of 77 candidates, totalling 385 fits\n",
      "finished Gridsearch trial 2 classifier 3\n",
      "\n",
      "\n",
      "Fitting 5 folds for each of 77 candidates, totalling 385 fits\n",
      "finished Gridsearch trial 3 classifier 3\n",
      "\n",
      "\n",
      "Fitting 5 folds for each of 77 candidates, totalling 385 fits\n",
      "finished Gridsearch trial 4 classifier 3\n",
      "\n",
      "\n",
      "Fitting 5 folds for each of 77 candidates, totalling 385 fits\n",
      "finished Gridsearch trial 5 classifier 3\n",
      "\n",
      "\n",
      "Fitting 5 folds for each of 42 candidates, totalling 210 fits\n",
      "finished Gridsearch trial 1 classifier 4\n",
      "\n",
      "\n",
      "Fitting 5 folds for each of 42 candidates, totalling 210 fits\n",
      "finished Gridsearch trial 2 classifier 4\n",
      "\n",
      "\n",
      "Fitting 5 folds for each of 42 candidates, totalling 210 fits\n",
      "finished Gridsearch trial 3 classifier 4\n",
      "\n",
      "\n",
      "Fitting 5 folds for each of 42 candidates, totalling 210 fits\n",
      "finished Gridsearch trial 4 classifier 4\n",
      "\n",
      "\n",
      "Fitting 5 folds for each of 42 candidates, totalling 210 fits\n",
      "finished Gridsearch trial 5 classifier 4\n",
      "\n",
      "\n",
      "Fitting 5 folds for each of 432 candidates, totalling 2160 fits\n",
      "finished Gridsearch trial 1 classifier 5\n",
      "\n",
      "\n",
      "Fitting 5 folds for each of 432 candidates, totalling 2160 fits\n",
      "finished Gridsearch trial 2 classifier 5\n",
      "\n",
      "\n",
      "Fitting 5 folds for each of 432 candidates, totalling 2160 fits\n",
      "finished Gridsearch trial 3 classifier 5\n",
      "\n",
      "\n",
      "Fitting 5 folds for each of 432 candidates, totalling 2160 fits\n",
      "finished Gridsearch trial 4 classifier 5\n",
      "\n",
      "\n",
      "Fitting 5 folds for each of 432 candidates, totalling 2160 fits\n",
      "finished Gridsearch trial 5 classifier 5\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "%%capture --no-stdout --no-display\n",
    "best_clf_list, trial_storage, training_perf = experiment()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Calculating and Organizing Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best Models On Average For Test Set:\n",
      "{'Model #': 1, 'average': 0.7397841712938319, 'accuracy': 0.7155, 'roc_auc_score': 0.779130694820054, 'f1_score': 0.7247218190614417}\n",
      "{'Model #': 2, 'average': 0.6740052911361613, 'accuracy': 0.654, 'roc_auc_score': 0.7072315596829936, 'f1_score': 0.6607843137254902}\n",
      "{'Model #': 3, 'average': 0.730361630281657, 'accuracy': 0.711, 'roc_auc_score': 0.7731883391208333, 'f1_score': 0.7068965517241379}\n",
      "{'Model #': 4, 'average': 0.6369932174616385, 'accuracy': 0.597, 'roc_auc_score': 0.6237413741374138, 'f1_score': 0.690238278247502}\n",
      "{'Model #': 5, 'average': 0.6760133373701054, 'accuracy': 0.65, 'roc_auc_score': 0.712968241775388, 'f1_score': 0.6650717703349281}\n",
      "\n",
      "Train Set Data\n",
      "[{'Model #': 1, 'average': 0.9896706077254435, 'accuracy': 0.986, 'roc_auc_score': 0.9968458152711918, 'f1 score': 0.9861660079051385}, {'Model #': 1, 'average': 0.9896706077254435, 'accuracy': 0.986, 'roc_auc_score': 0.9968458152711918, 'f1_score': 0.9861660079051385}, {'Model #': 1, 'average': 0.9885347265654086, 'accuracy': 0.9844, 'roc_auc_score': 0.9966191994590718, 'f1 score': 0.9845849802371541}, {'Model #': 1, 'average': 0.9885347265654086, 'accuracy': 0.9844, 'roc_auc_score': 0.9966191994590718, 'f1_score': 0.9845849802371541}, {'Model #': 1, 'average': 0.9911563027776232, 'accuracy': 0.988, 'roc_auc_score': 0.9973783962248906, 'f1 score': 0.9880905121079794}, {'Model #': 1, 'average': 0.9911563027776232, 'accuracy': 0.988, 'roc_auc_score': 0.9973783962248906, 'f1_score': 0.9880905121079794}, {'Model #': 1, 'average': 0.9890587078721288, 'accuracy': 0.985, 'roc_auc_score': 0.9972152252807137, 'f1 score': 0.9849608983356727}, {'Model #': 1, 'average': 0.9890587078721288, 'accuracy': 0.985, 'roc_auc_score': 0.9972152252807137, 'f1_score': 0.9849608983356727}, {'Model #': 1, 'average': 0.9893414100594601, 'accuracy': 0.9856, 'roc_auc_score': 0.996692957883493, 'f1 score': 0.9857312722948871}, {'Model #': 1, 'average': 0.9893414100594601, 'accuracy': 0.9856, 'roc_auc_score': 0.996692957883493, 'f1_score': 0.9857312722948871}]\n",
      "[{'Model #': 2, 'average': 0.72469402050915, 'accuracy': 0.6968, 'roc_auc_score': 0.7646209546282842, 'f1 score': 0.7126611068991658}, {'Model #': 2, 'average': 0.72469402050915, 'accuracy': 0.6968, 'roc_auc_score': 0.7646209546282842, 'f1_score': 0.7126611068991658}, {'Model #': 2, 'average': 0.7291600932540728, 'accuracy': 0.6988, 'roc_auc_score': 0.7734760437561671, 'f1 score': 0.7152042360060515}, {'Model #': 2, 'average': 0.7291600932540728, 'accuracy': 0.6988, 'roc_auc_score': 0.7734760437561671, 'f1_score': 0.7152042360060515}, {'Model #': 2, 'average': 0.7335154491888453, 'accuracy': 0.7012, 'roc_auc_score': 0.7766291983460455, 'f1 score': 0.7227171492204899}, {'Model #': 2, 'average': 0.7335154491888453, 'accuracy': 0.7012, 'roc_auc_score': 0.7766291983460455, 'f1_score': 0.7227171492204899}, {'Model #': 2, 'average': 0.6731406149895767, 'accuracy': 0.6436, 'roc_auc_score': 0.7069255238984957, 'f1 score': 0.6688963210702341}, {'Model #': 2, 'average': 0.6731406149895767, 'accuracy': 0.6436, 'roc_auc_score': 0.7069255238984957, 'f1_score': 0.6688963210702341}, {'Model #': 2, 'average': 0.7282117122283416, 'accuracy': 0.6944, 'roc_auc_score': 0.7776692977083506, 'f1 score': 0.7125658389766741}, {'Model #': 2, 'average': 0.7282117122283416, 'accuracy': 0.6944, 'roc_auc_score': 0.7776692977083506, 'f1_score': 0.7125658389766741}]\n",
      "[{'Model #': 3, 'average': 0.7826221272322312, 'accuracy': 0.7572, 'roc_auc_score': 0.841596518116702, 'f1 score': 0.7490698635799917}, {'Model #': 3, 'average': 0.7826221272322312, 'accuracy': 0.7572, 'roc_auc_score': 0.841596518116702, 'f1_score': 0.7490698635799917}, {'Model #': 3, 'average': 0.7513417504283962, 'accuracy': 0.7274, 'roc_auc_score': 0.805300368848059, 'f1 score': 0.7213248824371296}, {'Model #': 3, 'average': 0.7513417504283962, 'accuracy': 0.7274, 'roc_auc_score': 0.805300368848059, 'f1_score': 0.7213248824371296}, {'Model #': 3, 'average': 0.7759264666244775, 'accuracy': 0.7522, 'roc_auc_score': 0.8346218418554523, 'f1 score': 0.7409575580179804}, {'Model #': 3, 'average': 0.7759264666244775, 'accuracy': 0.7522, 'roc_auc_score': 0.8346218418554523, 'f1_score': 0.7409575580179804}, {'Model #': 3, 'average': 0.7474944978213438, 'accuracy': 0.7214, 'roc_auc_score': 0.804732444797772, 'f1 score': 0.7163510486662594}, {'Model #': 3, 'average': 0.7474944978213438, 'accuracy': 0.7214, 'roc_auc_score': 0.804732444797772, 'f1_score': 0.7163510486662594}, {'Model #': 3, 'average': 0.6902500562681393, 'accuracy': 0.666, 'roc_auc_score': 0.7171332589652858, 'f1 score': 0.687616909839132}, {'Model #': 3, 'average': 0.6902500562681393, 'accuracy': 0.666, 'roc_auc_score': 0.7171332589652858, 'f1_score': 0.687616909839132}]\n",
      "[{'Model #': 4, 'average': 0.6322166932706624, 'accuracy': 0.5952, 'roc_auc_score': 0.6096108593491373, 'f1 score': 0.6918392204628502}, {'Model #': 4, 'average': 0.6322166932706624, 'accuracy': 0.5952, 'roc_auc_score': 0.6096108593491373, 'f1_score': 0.6918392204628502}, {'Model #': 4, 'average': 0.6276307861869385, 'accuracy': 0.5938, 'roc_auc_score': 0.601121856179497, 'f1 score': 0.6879705023813181}, {'Model #': 4, 'average': 0.6276307861869383, 'accuracy': 0.5938, 'roc_auc_score': 0.601121856179497, 'f1_score': 0.6879705023813181}, {'Model #': 4, 'average': 0.6170462399940608, 'accuracy': 0.575, 'roc_auc_score': 0.6036159892070245, 'f1 score': 0.672522730775158}, {'Model #': 4, 'average': 0.6170462399940608, 'accuracy': 0.575, 'roc_auc_score': 0.6036159892070245, 'f1_score': 0.672522730775158}, {'Model #': 4, 'average': 0.6108843825400015, 'accuracy': 0.5884, 'roc_auc_score': 0.6069602325688944, 'f1 score': 0.6372929150511104}, {'Model #': 4, 'average': 0.6108843825400015, 'accuracy': 0.5884, 'roc_auc_score': 0.6069602325688944, 'f1_score': 0.6372929150511104}, {'Model #': 4, 'average': 0.6307999905904254, 'accuracy': 0.59, 'roc_auc_score': 0.6164318345163742, 'f1 score': 0.685968137254902}, {'Model #': 4, 'average': 0.6307999905904254, 'accuracy': 0.59, 'roc_auc_score': 0.6164318345163742, 'f1_score': 0.685968137254902}]\n",
      "[{'Model #': 5, 'average': 0.6813977289707274, 'accuracy': 0.6492, 'roc_auc_score': 0.6913771003964672, 'f1 score': 0.7036160865157147}, {'Model #': 5, 'average': 0.6813977289707274, 'accuracy': 0.6492, 'roc_auc_score': 0.6913771003964672, 'f1_score': 0.7036160865157147}, {'Model #': 5, 'average': 0.6415233205897778, 'accuracy': 0.617, 'roc_auc_score': 0.6696421871427499, 'f1 score': 0.6379277746265835}, {'Model #': 5, 'average': 0.6415233205897778, 'accuracy': 0.617, 'roc_auc_score': 0.6696421871427499, 'f1_score': 0.6379277746265835}, {'Model #': 5, 'average': 0.6511876039768523, 'accuracy': 0.6126, 'roc_auc_score': 0.6584517481705174, 'f1 score': 0.6825110637600393}, {'Model #': 5, 'average': 0.6511876039768523, 'accuracy': 0.6126, 'roc_auc_score': 0.6584517481705174, 'f1_score': 0.6825110637600393}, {'Model #': 5, 'average': 0.699344575447375, 'accuracy': 0.6722, 'roc_auc_score': 0.7397582618717176, 'f1 score': 0.6860754644704079}, {'Model #': 5, 'average': 0.699344575447375, 'accuracy': 0.6722, 'roc_auc_score': 0.7397582618717176, 'f1_score': 0.6860754644704079}, {'Model #': 5, 'average': 0.6085182302322812, 'accuracy': 0.6004, 'roc_auc_score': 0.6939486841271578, 'f1 score': 0.5312060065696856}, {'Model #': 5, 'average': 0.6085182302322812, 'accuracy': 0.6004, 'roc_auc_score': 0.6939486841271578, 'f1_score': 0.5312060065696856}]\n",
      "\n",
      "set of averages of algorithms over 5 trials:\n",
      "{'1': [0.7397841712938319, 0.7295932348050758, 0.7364385320712664, 0.7257956155589397, 0.7257140550019482], '2': [0.6676176613693542, 0.6740052911361613, 0.6641355502586234, 0.6507480056613185, 0.672099431323765], '3': [0.730361630281657, 0.725492978749458, 0.7160258784928986, 0.7146393832181008, 0.6791102285801099], '4': [0.6284974285229111, 0.6210954610550077, 0.6115011492605995, 0.5917756804544217, 0.6369932174616385], '5': [0.6559063204500494, 0.6234998640953213, 0.6398691869128804, 0.6760133373701054, 0.6003038535357116]}\n",
      "\n",
      "set of acc values of algorithms over 5 trials\n",
      "{'1': [0.7155, 0.708, 0.7125, 0.697, 0.6975], '2': [0.644, 0.654, 0.633, 0.622, 0.651], '3': [0.711, 0.708, 0.696, 0.693, 0.657], '4': [0.594, 0.588, 0.566, 0.568, 0.597], '5': [0.623, 0.605, 0.602, 0.65, 0.597]}\n",
      "\n",
      "set of roc values of algorithms over 5 trials\n",
      "{'1': [0.779130694820054, 0.7724879961235188, 0.7850361475922452, 0.7815796896191654, 0.7781128112811282], '2': [0.6924987198156535, 0.7072315596829936, 0.6970331457160724, 0.6805683914047154, 0.6999099909990999], '3': [0.7731883391208333, 0.7814478537188991, 0.7744530331457161, 0.7610191597553122, 0.7076207620762076], '4': [0.5947036373237746, 0.5966592380792196, 0.604416760475297, 0.5862744097843178, 0.6237413741374138], '5': [0.6623263749979996, 0.6489947379170321, 0.6454494058786743, 0.712968241775388, 0.684818481848185]}\n",
      "\n",
      "set of f1 values of algorithms over 5 trials\n",
      "{'1': [0.7247218190614417, 0.7082917082917084, 0.711779448621554, 0.6988071570576541, 0.7015293537247164], '2': [0.6663542642924087, 0.6607843137254902, 0.6623735050597975, 0.6496756255792401, 0.6653883029721955], '3': [0.7068965517241379, 0.6870310825294748, 0.6776246023329798, 0.68989898989899, 0.6727099236641222], '4': [0.6967886482449589, 0.6786271450858035, 0.6640866873065016, 0.6210526315789474, 0.690238278247502], '5': [0.6823925863521483, 0.616504854368932, 0.6721581548599671, 0.6650717703349281, 0.5190930787589499]}\n"
     ]
    }
   ],
   "source": [
    "print('Best Models On Average For Test Set:')\n",
    "for element in best_clf_list:\n",
    "    print(element)\n",
    "\n",
    "print()\n",
    "\n",
    "print('Train Set Data')\n",
    "for i in range(len(training_perf)):\n",
    "    print(training_perf[str(i + 1)])\n",
    "\n",
    "print()\n",
    "\n",
    "alg_avg = {}\n",
    "alg_acc = {}\n",
    "alg_roc = {}\n",
    "alg_f1 = {}\n",
    "\n",
    "for i in range(len(trial_storage)):\n",
    "    alg_avg[str(i + 1)]=[]\n",
    "    alg_acc[str(i + 1)]=[]\n",
    "    alg_roc[str(i + 1)]=[]\n",
    "    alg_f1[str(i + 1)]=[]\n",
    "    for entry in trial_storage[str(i + 1)]:\n",
    "        alg_avg[str(i + 1)].append(entry['average'])\n",
    "        alg_acc[str(i + 1)].append(entry['accuracy'])\n",
    "        alg_roc[str(i + 1)].append(entry['roc_auc_score'])\n",
    "        alg_f1[str(i + 1)].append(entry['f1_score'])\n",
    "\n",
    "print('set of averages of algorithms over 5 trials:')\n",
    "print(alg_avg)\n",
    "print()\n",
    "print('set of acc values of algorithms over 5 trials')\n",
    "print(alg_acc)\n",
    "print()\n",
    "print('set of roc values of algorithms over 5 trials')\n",
    "print(alg_roc)\n",
    "print()\n",
    "print('set of f1 values of algorithms over 5 trials')\n",
    "print(alg_f1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'1': 0.7061, '2': 0.6407999999999999, '3': 0.6930000000000001, '4': 0.5826, '5': 0.6154}\n"
     ]
    }
   ],
   "source": [
    "# calculate average acc metric scores per algorithm over 5 trials\n",
    "alg_acc_averages = {}\n",
    "for i in range(len(alg_acc)):\n",
    "    alg_acc_averages[str(i + 1)] = sum(alg_acc[str(i + 1)])/5\n",
    "\n",
    "print(alg_acc_averages)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'1': 0.7792694678872223, '2': 0.695448361523707, '3': 0.7595458295633938, '4': 0.6011590839600045, '5': 0.6709114484834557}\n"
     ]
    }
   ],
   "source": [
    "# calculate average roc metric scores per algorithm over 5 trials\n",
    "alg_roc_averages = {}\n",
    "for i in range(len(alg_roc)):\n",
    "    alg_roc_averages[str(i + 1)] = sum(alg_roc[str(i + 1)])/5\n",
    "\n",
    "print(alg_roc_averages)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'1': 0.7090258973514149, '2': 0.6609152023258265, '3': 0.686832230029941, '4': 0.6701586780927427, '5': 0.6310440889349851}\n"
     ]
    }
   ],
   "source": [
    "# calculate average f1 metric scores per algorithm over 5 trials\n",
    "alg_f1_averages = {}\n",
    "for i in range(len(alg_f1)):\n",
    "    alg_f1_averages[str(i + 1)] = sum(alg_f1[str(i + 1)])/5\n",
    "\n",
    "print(alg_f1_averages)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'1': 0.7314651217462124, '2': 0.6657211879498445, '3': 0.7131260198644448, '4': 0.6179725873509158, '5': 0.6391185124728136}\n"
     ]
    }
   ],
   "source": [
    "averages = {}\n",
    "for i in range(len(alg_acc_averages)):\n",
    "    averages[str(i + 1)] = (alg_acc_averages[str(i + 1)] + alg_roc_averages[str(i + 1)] + alg_f1_averages[str(i + 1)])/3\n",
    "\n",
    "print(averages)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ttest_relResult(statistic=13.536554913411994, pvalue=1.9634473848890334e-09)\n",
      "Ttest_relResult(statistic=3.626745938180636, pvalue=0.002748710807162354)\n",
      "Ttest_relResult(statistic=7.091129109155958, pvalue=5.411836315428835e-06)\n",
      "Ttest_relResult(statistic=8.681118334491316, pvalue=5.224431932512865e-07)\n"
     ]
    }
   ],
   "source": [
    "# t-test best against rest mean of metrics (RF against rest)\n",
    "combined_metrics_1 = []\n",
    "combined_metrics_2 = []\n",
    "combined_metrics_3 = []\n",
    "combined_metrics_4 = []\n",
    "combined_metrics_5 = []\n",
    "\n",
    "for item in alg_acc['1']:\n",
    "    combined_metrics_1.append(item)\n",
    "for item in alg_roc['1']:\n",
    "    combined_metrics_1.append(item)\n",
    "for item in alg_f1['1']:\n",
    "    combined_metrics_1.append(item)\n",
    "    \n",
    "for item in alg_acc['2']:\n",
    "    combined_metrics_2.append(item)\n",
    "for item in alg_roc['2']:\n",
    "    combined_metrics_2.append(item)\n",
    "for item in alg_f1['2']:\n",
    "    combined_metrics_2.append(item)\n",
    "\n",
    "for item in alg_acc['3']:\n",
    "    combined_metrics_3.append(item)\n",
    "for item in alg_roc['3']:\n",
    "    combined_metrics_3.append(item)\n",
    "for item in alg_f1['3']:\n",
    "    combined_metrics_3.append(item)\n",
    "    \n",
    "for item in alg_acc['4']:\n",
    "    combined_metrics_4.append(item)\n",
    "for item in alg_roc['4']:\n",
    "    combined_metrics_4.append(item)\n",
    "for item in alg_f1['4']:\n",
    "    combined_metrics_4.append(item)\n",
    "\n",
    "for item in alg_acc['5']:\n",
    "    combined_metrics_5.append(item)\n",
    "for item in alg_roc['5']:\n",
    "    combined_metrics_5.append(item)\n",
    "for item in alg_f1['5']:\n",
    "    combined_metrics_5.append(item)\n",
    "    \n",
    "print(ttest_rel(combined_metrics_1, combined_metrics_2))\n",
    "print(ttest_rel(combined_metrics_1, combined_metrics_3))\n",
    "print(ttest_rel(combined_metrics_1, combined_metrics_4))\n",
    "print(ttest_rel(combined_metrics_1, combined_metrics_5))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {},
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}