feature-engineering.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Feature engineering\n",
    "\n",
    "In this notebook, we carry out the initial cleaning of the data. \n",
    "\n",
    "Overall, due to the relatively small number of samples in the training dataset, we aim to reduce the dimensionality of the data by:\n",
    "1. Eliminating features with too many **missing values**\n",
    "2. **Grouping** categorical variables that have a low count\n",
    "3. Eliminating features with **little information**, such as those that are mostly constant\n",
    "\n",
    "We also prepare the text data by vectorizing it."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.preprocessing import MultiLabelBinarizer\n",
    "from sklearn.feature_extraction.text import CountVectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 6000 entries, 9011 to 1625\n",
      "Data columns (total 51 columns):\n",
      " #   Column                    Non-Null Count  Dtype \n",
      "---  ------                    --------------  ----- \n",
      " 0   race                      6000 non-null   object\n",
      " 1   gender                    6000 non-null   object\n",
      " 2   age                       6000 non-null   object\n",
      " 3   weight                    6000 non-null   object\n",
      " 4   admission_type_id         6000 non-null   object\n",
      " 5   discharge_disposition_id  6000 non-null   object\n",
      " 6   admission_source_id       6000 non-null   object\n",
      " 7   time_in_hospital          6000 non-null   int64 \n",
      " 8   payer_code                6000 non-null   object\n",
      " 9   medical_specialty         6000 non-null   object\n",
      " 10  num_lab_procedures        6000 non-null   int64 \n",
      " 11  num_procedures            6000 non-null   int64 \n",
      " 12  num_medications           6000 non-null   int64 \n",
      " 13  number_outpatient         6000 non-null   int64 \n",
      " 14  number_emergency          6000 non-null   int64 \n",
      " 15  number_inpatient          6000 non-null   int64 \n",
      " 16  diag_1                    6000 non-null   object\n",
      " 17  diag_2                    6000 non-null   object\n",
      " 18  diag_3                    6000 non-null   object\n",
      " 19  number_diagnoses          6000 non-null   int64 \n",
      " 20  max_glu_serum             6000 non-null   object\n",
      " 21  A1Cresult                 6000 non-null   object\n",
      " 22  metformin                 6000 non-null   object\n",
      " 23  repaglinide               6000 non-null   object\n",
      " 24  nateglinide               6000 non-null   object\n",
      " 25  chlorpropamide            6000 non-null   object\n",
      " 26  glimepiride               6000 non-null   object\n",
      " 27  acetohexamide             6000 non-null   object\n",
      " 28  glipizide                 6000 non-null   object\n",
      " 29  glyburide                 6000 non-null   object\n",
      " 30  tolbutamide               6000 non-null   object\n",
      " 31  pioglitazone              6000 non-null   object\n",
      " 32  rosiglitazone             6000 non-null   object\n",
      " 33  acarbose                  6000 non-null   object\n",
      " 34  miglitol                  6000 non-null   object\n",
      " 35  troglitazone              6000 non-null   object\n",
      " 36  tolazamide                6000 non-null   object\n",
      " 37  examide                   6000 non-null   object\n",
      " 38  citoglipton               6000 non-null   object\n",
      " 39  insulin                   6000 non-null   object\n",
      " 40  glyburide.metformin       6000 non-null   object\n",
      " 41  glipizide.metformin       6000 non-null   object\n",
      " 42  glimepiride.pioglitazone  6000 non-null   object\n",
      " 43  metformin.rosiglitazone   6000 non-null   object\n",
      " 44  metformin.pioglitazone    6000 non-null   object\n",
      " 45  change                    6000 non-null   object\n",
      " 46  diabetesMed               6000 non-null   object\n",
      " 47  readmitted                6000 non-null   int64 \n",
      " 48  diag_1_desc               6000 non-null   object\n",
      " 49  diag_2_desc               6000 non-null   object\n",
      " 50  diag_3_desc               6000 non-null   object\n",
      "dtypes: int64(9), object(42)\n",
      "memory usage: 2.4+ MB\n"
     ]
    }
   ],
   "source": [
    "def read_data(filename, data_dir='data'):\n",
    "    df = pd.read_csv(os.path.join(data_dir, filename), index_col=0)\n",
    "    df.fillna('?', inplace=True)\n",
    "    return df\n",
    "\n",
    "df_train = read_data('diab_train.csv')\n",
    "df_val = read_data('diab_validation.csv')\n",
    "df_test = read_data('diab_test.csv')\n",
    "df_train.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>race</th>\n",
       "      <th>gender</th>\n",
       "      <th>age</th>\n",
       "      <th>weight</th>\n",
       "      <th>admission_type_id</th>\n",
       "      <th>discharge_disposition_id</th>\n",
       "      <th>admission_source_id</th>\n",
       "      <th>time_in_hospital</th>\n",
       "      <th>payer_code</th>\n",
       "      <th>medical_specialty</th>\n",
       "      <th>...</th>\n",
       "      <th>glipizide.metformin</th>\n",
       "      <th>glimepiride.pioglitazone</th>\n",
       "      <th>metformin.rosiglitazone</th>\n",
       "      <th>metformin.pioglitazone</th>\n",
       "      <th>change</th>\n",
       "      <th>diabetesMed</th>\n",
       "      <th>readmitted</th>\n",
       "      <th>diag_1_desc</th>\n",
       "      <th>diag_2_desc</th>\n",
       "      <th>diag_3_desc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>9011</th>\n",
       "      <td>AfricanAmerican</td>\n",
       "      <td>Male</td>\n",
       "      <td>[60-70)</td>\n",
       "      <td>?</td>\n",
       "      <td>Emergency</td>\n",
       "      <td>Discharged to home</td>\n",
       "      <td>Emergency Room</td>\n",
       "      <td>4</td>\n",
       "      <td>MC</td>\n",
       "      <td>?</td>\n",
       "      <td>...</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>0</td>\n",
       "      <td>Acute pericarditis in diseases classified else...</td>\n",
       "      <td>Secondary malignant neoplasm of kidney</td>\n",
       "      <td>Congestive heart failure, unspecified</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9569</th>\n",
       "      <td>Caucasian</td>\n",
       "      <td>Female</td>\n",
       "      <td>[70-80)</td>\n",
       "      <td>?</td>\n",
       "      <td>Elective</td>\n",
       "      <td>?</td>\n",
       "      <td>Physician Referral</td>\n",
       "      <td>1</td>\n",
       "      <td>?</td>\n",
       "      <td>Family/GeneralPractice</td>\n",
       "      <td>...</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>Ch</td>\n",
       "      <td>Yes</td>\n",
       "      <td>1</td>\n",
       "      <td>Malignant essential hypertension</td>\n",
       "      <td>Spinal stenosis, unspecified region</td>\n",
       "      <td>Diabetes mellitus without mention of complicat...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6090</th>\n",
       "      <td>Caucasian</td>\n",
       "      <td>Female</td>\n",
       "      <td>[80-90)</td>\n",
       "      <td>?</td>\n",
       "      <td>Urgent</td>\n",
       "      <td>Discharged/transferred to SNF</td>\n",
       "      <td>Emergency Room</td>\n",
       "      <td>2</td>\n",
       "      <td>MC</td>\n",
       "      <td>Emergency/Trauma</td>\n",
       "      <td>...</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>Ch</td>\n",
       "      <td>Yes</td>\n",
       "      <td>0</td>\n",
       "      <td>Urinary tract infection, site not specified</td>\n",
       "      <td>Streptococcus infection in conditions classifi...</td>\n",
       "      <td>Congestive heart failure, unspecified</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4282</th>\n",
       "      <td>AfricanAmerican</td>\n",
       "      <td>Female</td>\n",
       "      <td>[50-60)</td>\n",
       "      <td>?</td>\n",
       "      <td>Emergency</td>\n",
       "      <td>Discharged to home</td>\n",
       "      <td>Emergency Room</td>\n",
       "      <td>4</td>\n",
       "      <td>DM</td>\n",
       "      <td>?</td>\n",
       "      <td>...</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>Ch</td>\n",
       "      <td>Yes</td>\n",
       "      <td>1</td>\n",
       "      <td>Respiratory abnormality, unspecified</td>\n",
       "      <td>Hypertensive chronic kidney disease, malignant...</td>\n",
       "      <td>Diabetes mellitus without mention of complicat...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9084</th>\n",
       "      <td>Caucasian</td>\n",
       "      <td>Male</td>\n",
       "      <td>[80-90)</td>\n",
       "      <td>?</td>\n",
       "      <td>Elective</td>\n",
       "      <td>?</td>\n",
       "      <td>Physician Referral</td>\n",
       "      <td>13</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>...</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>Ch</td>\n",
       "      <td>Yes</td>\n",
       "      <td>1</td>\n",
       "      <td>Coronary atherosclerosis of unspecified type o...</td>\n",
       "      <td>Chronic airway obstruction, not elsewhere clas...</td>\n",
       "      <td>Malignant essential hypertension</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1776</th>\n",
       "      <td>Caucasian</td>\n",
       "      <td>Female</td>\n",
       "      <td>[70-80)</td>\n",
       "      <td>?</td>\n",
       "      <td>Emergency</td>\n",
       "      <td>Discharged/transferred to SNF</td>\n",
       "      <td>Emergency Room</td>\n",
       "      <td>3</td>\n",
       "      <td>?</td>\n",
       "      <td>InternalMedicine</td>\n",
       "      <td>...</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>Yes</td>\n",
       "      <td>0</td>\n",
       "      <td>Cerebral thrombosis without mention of cerebra...</td>\n",
       "      <td>Urinary tract infection, site not specified</td>\n",
       "      <td>Diabetes mellitus without mention of complicat...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>797</th>\n",
       "      <td>Caucasian</td>\n",
       "      <td>Male</td>\n",
       "      <td>[60-70)</td>\n",
       "      <td>[75-100)</td>\n",
       "      <td>Elective</td>\n",
       "      <td>Discharged/transferred to ICF</td>\n",
       "      <td>Transfer from another health care facility</td>\n",
       "      <td>5</td>\n",
       "      <td>MC</td>\n",
       "      <td>InternalMedicine</td>\n",
       "      <td>...</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>0</td>\n",
       "      <td>Acquired hypertrophic pyloric stenosis</td>\n",
       "      <td>Congestive heart failure, unspecified</td>\n",
       "      <td>Paroxysmal supraventricular tachycardia</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>413</th>\n",
       "      <td>Caucasian</td>\n",
       "      <td>Female</td>\n",
       "      <td>[60-70)</td>\n",
       "      <td>?</td>\n",
       "      <td>Emergency</td>\n",
       "      <td>Discharged to home</td>\n",
       "      <td>Transfer from a hospital</td>\n",
       "      <td>3</td>\n",
       "      <td>MC</td>\n",
       "      <td>InternalMedicine</td>\n",
       "      <td>...</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>Yes</td>\n",
       "      <td>0</td>\n",
       "      <td>Basilar artery syndrome</td>\n",
       "      <td>Malignant hypertensive heart disease without h...</td>\n",
       "      <td>Congestive heart failure, unspecified</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8150</th>\n",
       "      <td>Caucasian</td>\n",
       "      <td>Male</td>\n",
       "      <td>[20-30)</td>\n",
       "      <td>?</td>\n",
       "      <td>Emergency</td>\n",
       "      <td>Discharged to home</td>\n",
       "      <td>Emergency Room</td>\n",
       "      <td>2</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>...</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>0</td>\n",
       "      <td>Hyperosmolality and/or hypernatremia</td>\n",
       "      <td>Adenovirus infection in conditions classified ...</td>\n",
       "      <td>Personal history of allergy, other than to med...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5317</th>\n",
       "      <td>Asian</td>\n",
       "      <td>Male</td>\n",
       "      <td>[80-90)</td>\n",
       "      <td>?</td>\n",
       "      <td>Urgent</td>\n",
       "      <td>Discharged/transferred to another short term h...</td>\n",
       "      <td>Emergency Room</td>\n",
       "      <td>1</td>\n",
       "      <td>SP</td>\n",
       "      <td>Family/GeneralPractice</td>\n",
       "      <td>...</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>Ch</td>\n",
       "      <td>Yes</td>\n",
       "      <td>1</td>\n",
       "      <td>Closed fracture of acetabulum</td>\n",
       "      <td>Emphysematous bleb</td>\n",
       "      <td>Closed fracture of unspecified part of upper e...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10 rows × 51 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                 race  gender      age    weight admission_type_id  \\\n",
       "9011  AfricanAmerican    Male  [60-70)         ?         Emergency   \n",
       "9569        Caucasian  Female  [70-80)         ?          Elective   \n",
       "6090        Caucasian  Female  [80-90)         ?            Urgent   \n",
       "4282  AfricanAmerican  Female  [50-60)         ?         Emergency   \n",
       "9084        Caucasian    Male  [80-90)         ?          Elective   \n",
       "1776        Caucasian  Female  [70-80)         ?         Emergency   \n",
       "797         Caucasian    Male  [60-70)  [75-100)          Elective   \n",
       "413         Caucasian  Female  [60-70)         ?         Emergency   \n",
       "8150        Caucasian    Male  [20-30)         ?         Emergency   \n",
       "5317            Asian    Male  [80-90)         ?            Urgent   \n",
       "\n",
       "                               discharge_disposition_id  \\\n",
       "9011                                 Discharged to home   \n",
       "9569                                                  ?   \n",
       "6090                      Discharged/transferred to SNF   \n",
       "4282                                 Discharged to home   \n",
       "9084                                                  ?   \n",
       "1776                      Discharged/transferred to SNF   \n",
       "797                       Discharged/transferred to ICF   \n",
       "413                                  Discharged to home   \n",
       "8150                                 Discharged to home   \n",
       "5317  Discharged/transferred to another short term h...   \n",
       "\n",
       "                             admission_source_id  time_in_hospital payer_code  \\\n",
       "9011                              Emergency Room                 4         MC   \n",
       "9569                          Physician Referral                 1          ?   \n",
       "6090                              Emergency Room                 2         MC   \n",
       "4282                              Emergency Room                 4         DM   \n",
       "9084                          Physician Referral                13          ?   \n",
       "1776                              Emergency Room                 3          ?   \n",
       "797   Transfer from another health care facility                 5         MC   \n",
       "413                     Transfer from a hospital                 3         MC   \n",
       "8150                              Emergency Room                 2          ?   \n",
       "5317                              Emergency Room                 1         SP   \n",
       "\n",
       "           medical_specialty  ...  glipizide.metformin  \\\n",
       "9011                       ?  ...                   No   \n",
       "9569  Family/GeneralPractice  ...                   No   \n",
       "6090        Emergency/Trauma  ...                   No   \n",
       "4282                       ?  ...                   No   \n",
       "9084                       ?  ...                   No   \n",
       "1776        InternalMedicine  ...                   No   \n",
       "797         InternalMedicine  ...                   No   \n",
       "413         InternalMedicine  ...                   No   \n",
       "8150                       ?  ...                   No   \n",
       "5317  Family/GeneralPractice  ...                   No   \n",
       "\n",
       "      glimepiride.pioglitazone  metformin.rosiglitazone  \\\n",
       "9011                        No                       No   \n",
       "9569                        No                       No   \n",
       "6090                        No                       No   \n",
       "4282                        No                       No   \n",
       "9084                        No                       No   \n",
       "1776                        No                       No   \n",
       "797                         No                       No   \n",
       "413                         No                       No   \n",
       "8150                        No                       No   \n",
       "5317                        No                       No   \n",
       "\n",
       "      metformin.pioglitazone  change  diabetesMed readmitted  \\\n",
       "9011                      No      No           No          0   \n",
       "9569                      No      Ch          Yes          1   \n",
       "6090                      No      Ch          Yes          0   \n",
       "4282                      No      Ch          Yes          1   \n",
       "9084                      No      Ch          Yes          1   \n",
       "1776                      No      No          Yes          0   \n",
       "797                       No      No           No          0   \n",
       "413                       No      No          Yes          0   \n",
       "8150                      No      No           No          0   \n",
       "5317                      No      Ch          Yes          1   \n",
       "\n",
       "                                            diag_1_desc  \\\n",
       "9011  Acute pericarditis in diseases classified else...   \n",
       "9569                   Malignant essential hypertension   \n",
       "6090        Urinary tract infection, site not specified   \n",
       "4282               Respiratory abnormality, unspecified   \n",
       "9084  Coronary atherosclerosis of unspecified type o...   \n",
       "1776  Cerebral thrombosis without mention of cerebra...   \n",
       "797              Acquired hypertrophic pyloric stenosis   \n",
       "413                             Basilar artery syndrome   \n",
       "8150               Hyperosmolality and/or hypernatremia   \n",
       "5317                      Closed fracture of acetabulum   \n",
       "\n",
       "                                            diag_2_desc  \\\n",
       "9011             Secondary malignant neoplasm of kidney   \n",
       "9569                Spinal stenosis, unspecified region   \n",
       "6090  Streptococcus infection in conditions classifi...   \n",
       "4282  Hypertensive chronic kidney disease, malignant...   \n",
       "9084  Chronic airway obstruction, not elsewhere clas...   \n",
       "1776        Urinary tract infection, site not specified   \n",
       "797               Congestive heart failure, unspecified   \n",
       "413   Malignant hypertensive heart disease without h...   \n",
       "8150  Adenovirus infection in conditions classified ...   \n",
       "5317                                 Emphysematous bleb   \n",
       "\n",
       "                                            diag_3_desc  \n",
       "9011              Congestive heart failure, unspecified  \n",
       "9569  Diabetes mellitus without mention of complicat...  \n",
       "6090              Congestive heart failure, unspecified  \n",
       "4282  Diabetes mellitus without mention of complicat...  \n",
       "9084                   Malignant essential hypertension  \n",
       "1776  Diabetes mellitus without mention of complicat...  \n",
       "797             Paroxysmal supraventricular tachycardia  \n",
       "413               Congestive heart failure, unspecified  \n",
       "8150  Personal history of allergy, other than to med...  \n",
       "5317  Closed fracture of unspecified part of upper e...  \n",
       "\n",
       "[10 rows x 51 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.head(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###  Dealing with missing values\n",
    "The columns: **'weight'**, **'payer_code'** and **'medical_specialty'** have too many missing values to provide meaningful information to the machine learning algorithm, so we just remove them.\n",
    "\n",
    "For the other columns with a smaller number of missing values, we just leave the missing value as a '?'. In the categorical features, this would correspond to a new 'unknown' category. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Missing values per column\n",
      "race 124\n",
      "weight 5757\n",
      "admission_type_id 416\n",
      "discharge_disposition_id 283\n",
      "admission_source_id 551\n",
      "payer_code 3211\n",
      "medical_specialty 2440\n",
      "diag_1 1\n",
      "diag_2 34\n",
      "diag_3 122\n",
      "diag_1_desc 1\n",
      "diag_2_desc 34\n",
      "diag_3_desc 122\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/carlos/anaconda3/envs/covid/lib/python3.8/site-packages/pandas/core/ops/array_ops.py:253: FutureWarning: elementwise comparison failed; returning scalar instead, but in the future will perform elementwise comparison\n",
      "  res_values = method(rvalues)\n"
     ]
    }
   ],
   "source": [
    "print(\"Missing values per column\")\n",
    "for column in df_train.columns:\n",
    "    num_missing_values = (df_train[column] == '?').sum()\n",
    "    if num_missing_values > 0:\n",
    "        print(column, num_missing_values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "columns_to_remove = ['weight', 'payer_code', 'medical_specialty']\n",
    "df_train.drop(columns_to_remove, axis='columns', inplace=True)\n",
    "df_val.drop(columns_to_remove, axis='columns', inplace=True)\n",
    "df_test.drop(columns_to_remove, axis='columns', inplace=True)\n",
    "\n",
    "assert(not df_train.isna().values.any())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Feature engineering: age\n",
    "The younger age groups have too few data points, so we group them into one new age group to normalize the data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAEoCAYAAAC0OiEVAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAf70lEQVR4nO3de5hkVX3u8e/LjKCIwgAN6szgoBm8xgt2EENikJsgRkgiiieRiWImiRBMzDk6xJyDl+gz5HhC9ETNmcAoJAZERBkFBESBE+PADMgdgQkgNNf2DOIFow6+54+9mil6qntmuqp2bXu/n+epp3etvarWr6t2/2r3qrXXkm0iIqIdthl2ABERUZ8k/YiIFknSj4hokST9iIgWSdKPiGiRJP2IiBaZu7kKklYCrwcesv3ijvI/A44HNgDn235PKT8ROBZ4DDjB9kWl/FDgY8Ac4FTbyzfX9q677upFixZt7e8UEdFqV1999fdsj3Tbt9mkD3wG+AfgjIkCSa8BjgBeYvunknYr5S8EjgZeBDwL+JqkvcrDPgEcDIwBayStsn3zdA0vWrSItWvXbkGIERExQdJ3p9q32aRv+wpJiyYV/ymw3PZPS52HSvkRwFml/E5J64B9yr51tu8oAZ1V6k6b9CMior9m2qe/F/Cbkq6UdLmkXyvl84F7OuqNlbKpyiMiokZb0r0z1ePmAfsCvwacLek5gLrUNd0/XLrO/yBpKbAUYI899phheBER0c1Mz/THgHNduQr4BbBrKV/YUW8BcN805ZuwvcL2qO3RkZGu30NERMQMzTTpfwk4AKB8Ubst8D1gFXC0pO0k7QksBq4C1gCLJe0paVuqL3tX9Rp8RERsnS0ZsnkmsD+wq6Qx4CRgJbBS0o3Az4AlrqbrvEnS2VRf0G4AjrP9WHme44GLqIZsrrR90wB+n4iImIaaPLXy6OioM2QzImLrSLra9mi3fbkiNyKiRZL0IyJaZKZDNiMittiiZef3/Bx3LT+8D5FEzvQjIlokST8iokWS9CMiWiRJPyKiRZL0IyJaJEk/IqJFkvQjIlokST8iokWS9CMiWiRJPyKiRZL0IyJaJEk/IqJFkvQjIlokST8iokWS9CMiWmSzSV/SSkkPlfVwJ+/7r5IsaddyX5I+LmmdpOsl7d1Rd4mk28ttSX9/jYiI2BJbcqb/GeDQyYWSFgIHA3d3FB8GLC63pcCnSt2dqRZUfyWwD3CSpHm9BB4REVtvs0nf9hXA+i67TgHeA3SurH4EcIYrq4GdJD0TeC1wie31th8GLqHLB0lERAzWjPr0Jb0BuNf2dZN2zQfu6bg/VsqmKo+IiBpt9Rq5krYH3gcc0m13lzJPU97t+ZdSdQ2xxx57bG14ERExjZmc6T8X2BO4TtJdwALgGknPoDqDX9hRdwFw3zTlm7C9wvao7dGRkZEZhBcREVPZ6qRv+wbbu9leZHsRVULf2/YDwCrgmDKKZ1/gEdv3AxcBh0iaV77APaSURUREjbZkyOaZwLeA50kak3TsNNUvAO4A1gH/BLwTwPZ64EPAmnL7YCmLiIgabbZP3/ZbNrN/Uce2geOmqLcSWLmV8UVERB/lityIiBZJ0o+IaJEk/YiIFknSj4hoka2+OCsifrksWnZ+T4+/a/nhfYokmiBn+hERLZKkHxHRIkn6EREtkqQfEdEiSfoRES2SpB8R0SJJ+hERLZKkHxHRIkn6EREtkqQfEdEiSfoRES2SpB8R0SJJ+hERLbIla+SulPSQpBs7yv6npO9Iul7SFyXt1LHvREnrJN0q6bUd5YeWsnWSlvX/V4mIiM3ZkjP9zwCHTiq7BHix7ZcAtwEnAkh6IXA08KLymE9KmiNpDvAJ4DDghcBbSt2IiKjRZpO+7SuA9ZPKLra9odxdDSwo20cAZ9n+qe07gXXAPuW2zvYdtn8GnFXqRkREjfrRp/924MKyPR+4p2PfWCmbqnwTkpZKWitp7fj4eB/Ci4iICT0lfUnvAzYAn50o6lLN05RvWmivsD1qe3RkZKSX8CIiYpIZL5coaQnweuBA2xMJfAxY2FFtAXBf2Z6qPCIiajKjM31JhwLvBd5g+9GOXauAoyVtJ2lPYDFwFbAGWCxpT0nbUn3Zu6q30CMiYmtt9kxf0pnA/sCuksaAk6hG62wHXCIJYLXtP7F9k6SzgZupun2Os/1YeZ7jgYuAOcBK2zcN4PeJiIhpbDbp235Ll+LTpqn/YeDDXcovAC7YqugiIqKvckVuRESLJOlHRLRIkn5ERIsk6UdEtEiSfkREiyTpR0S0SJJ+RESLJOlHRLRIkn5ERIsk6UdEtEiSfkREiyTpR0S0SJJ+RESLJOlHRLRIkn5ERIsk6UdEtEiSfkREi2w26UtaKekhSTd2lO0s6RJJt5ef80q5JH1c0jpJ10vau+MxS0r928ui6hERUbPNLpcIfAb4B+CMjrJlwKW2l0taVu6/FziMajH0xcArgU8Br5S0M9XauqOAgaslrbL9cL9+kYimWbTs/J6f467lh/chkoiNNnumb/sKYP2k4iOA08v26cCRHeVnuLIa2EnSM4HXApfYXl8S/SXAof34BSIiYsvNtE9/d9v3A5Sfu5Xy+cA9HfXGStlU5RERUaN+f5GrLmWepnzTJ5CWSlorae34+Hhfg4uIaLuZJv0HS7cN5edDpXwMWNhRbwFw3zTlm7C9wvao7dGRkZEZhhcREd3MNOmvAiZG4CwBzusoP6aM4tkXeKR0/1wEHCJpXhnpc0gpi4iIGm129I6kM4H9gV0ljVGNwlkOnC3pWOBu4KhS/QLgdcA64FHgbQC210v6ELCm1Pug7clfDkdEDFSvI6pmw2iqzSZ922+ZYteBXeoaOG6K51kJrNyq6CIioq9yRW5ERIsk6UdEtEiSfkREiyTpR0S0SJJ+RESLJOlHRLRIkn5ERIsk6UdEtEiSfkREiyTpR0S0SJJ+RESLJOlHRLRIkn5ERIsk6UdEtEiSfkREiyTpR0S0SJJ+RESLJOlHRLRIT0lf0l9IuknSjZLOlPRkSXtKulLS7ZI+J2nbUne7cn9d2b+oH79ARERsuRknfUnzgROAUdsvBuYARwMnA6fYXgw8DBxbHnIs8LDtXwFOKfUiIqJGvXbvzAWeImkusD1wP3AAcE7ZfzpwZNk+otyn7D9QknpsPyIitsKMk77te4GPAndTJftHgKuB79veUKqNAfPL9nzgnvLYDaX+LjNtPyIitl4v3TvzqM7e9wSeBTwVOKxLVU88ZJp9nc+7VNJaSWvHx8dnGl5ERHTRS/fOQcCdtsdt/xw4F/h1YKfS3QOwALivbI8BCwHK/h2B9ZOf1PYK26O2R0dGRnoILyIiJusl6d8N7Ctp+9I3fyBwM/AN4I2lzhLgvLK9qtyn7P+67U3O9CMiYnB66dO/kuoL2WuAG8pzrQDeC7xb0jqqPvvTykNOA3Yp5e8GlvUQd0REzMDczVeZmu2TgJMmFd8B7NOl7n8CR/XSXkRE9CZX5EZEtEiSfkREiyTpR0S0SJJ+RESLJOlHRLRIkn5ERIsk6UdEtEiSfkREiyTpR0S0SJJ+RESLJOlHRLRIkn5ERIsk6UdEtEiSfkREiyTpR0S0SJJ+RESLJOlHRLRIkn5ERIv0lPQl7STpHEnfkXSLpFdJ2lnSJZJuLz/nlbqS9HFJ6yRdL2nv/vwKERGxpXo90/8Y8FXbzwdeCtxCteD5pbYXA5eycQH0w4DF5bYU+FSPbUdExFaacdKX9HTg1cBpALZ/Zvv7wBHA6aXa6cCRZfsI4AxXVgM7SXrmjCOPiIit1suZ/nOAceDTkr4t6VRJTwV2t30/QPm5W6k/H7in4/FjpSwiImrSS9KfC+wNfMr2y4Efs7Erpxt1KfMmlaSlktZKWjs+Pt5DeBERMVkvSX8MGLN9Zbl/DtWHwIMT3Tbl50Md9Rd2PH4BcN/kJ7W9wvao7dGRkZEewouIiMlmnPRtPwDcI+l5pehA4GZgFbCklC0Bzivbq4BjyiiefYFHJrqBIiKiHnN7fPyfAZ+VtC1wB/A2qg+SsyUdC9wNHFXqXgC8DlgHPFrqRkREjXpK+ravBUa77DqwS10Dx/XSXkRE9KbXM/2IRlq07PyeHn/X8sP7FElEs2QahoiIFknSj4hokST9iIgWSdKPiGiRJP2IiBZJ0o+IaJEk/YiIFknSj4hokST9iIgWSdKPiGiRJP2IiBZJ0o+IaJEk/YiIFknSj4hokST9iIgWSdKPiGiRJP2IiBbpOelLmiPp25K+Uu7vKelKSbdL+lxZPxdJ25X768r+Rb22HRERW6cfZ/rvAm7puH8ycIrtxcDDwLGl/FjgYdu/ApxS6kVERI16SvqSFgCHA6eW+wIOAM4pVU4HjizbR5T7lP0HlvoREVGTXs/0/x54D/CLcn8X4Pu2N5T7Y8D8sj0fuAeg7H+k1H8CSUslrZW0dnx8vMfwIiKi04yTvqTXAw/ZvrqzuEtVb8G+jQX2CtujtkdHRkZmGl5ERHQxt4fH7ge8QdLrgCcDT6c6899J0txyNr8AuK/UHwMWAmOS5gI7Aut7aD8iIrbSjM/0bZ9oe4HtRcDRwNdt/z7wDeCNpdoS4Lyyvarcp+z/uu1NzvQjImJwBjFO/73AuyWto+qzP62UnwbsUsrfDSwbQNsRETGNXrp3Hmf7MuCysn0HsE+XOv8JHNWP9iIiYmZyRW5ERIsk6UdEtEiSfkREiyTpR0S0SJJ+RESLJOlHRLRIkn5ERIsk6UdEtEiSfkREiyTpR0S0SJJ+RESLJOlHRLRIkn5ERIsk6UdEtEhfplaOmLBo2fk9P8ddyw/vQyQR0U3O9CMiWiRJPyKiRWac9CUtlPQNSbdIuknSu0r5zpIukXR7+TmvlEvSxyWtk3S9pL379UtERMSW6eVMfwPwl7ZfAOwLHCfphVRr315qezFwKRvXwj0MWFxuS4FP9dB2RETMwIyTvu37bV9Ttn8I3ALMB44ATi/VTgeOLNtHAGe4shrYSdIzZxx5RERstb706UtaBLwcuBLY3fb9UH0wALuVavOBezoeNlbKIiKiJj0nfUk7AF8A/tz2D6ar2qXMXZ5vqaS1ktaOj4/3Gl5ERHToKelLehJVwv+s7XNL8YMT3Tbl50OlfAxY2PHwBcB9k5/T9grbo7ZHR0ZGegkvIiIm6WX0joDTgFts/13HrlXAkrK9BDivo/yYMopnX+CRiW6giIioRy9X5O4HvBW4QdK1peyvgOXA2ZKOBe4Gjir7LgBeB6wDHgXe1kPbERExAzNO+rb/je799AAHdqlv4LiZthcREb3LFbkRES2SCdciImo07EkJc6YfEdEiSfoRES2SpB8R0SJJ+hERLZKkHxHRIkn6EREtkqQfEdEiSfoRES2SpB8R0SK5IncWGfaVfhHRfDnTj4hokST9iIgWSfdOH6RbJSJ+WeRMPyKiRZL0IyJaJEk/IqJFak/6kg6VdKukdZKW1d1+RESb1fpFrqQ5wCeAg4ExYI2kVbZvnulz5kvUiIgtV/eZ/j7AOtt32P4ZcBZwRM0xRES0Vt1Jfz5wT8f9sVIWERE1kO36GpOOAl5r+x3l/luBfWz/WUedpcDScvd5wK09Nrsr8L0en6MfmhBHE2KAZsTRhBigGXE0IQZoRhxNiAF6j+PZtke67aj74qwxYGHH/QXAfZ0VbK8AVvSrQUlrbY/26/l+meNoQgxNiaMJMTQljibE0JQ4mhDDoOOou3tnDbBY0p6StgWOBlbVHENERGvVeqZve4Ok44GLgDnASts31RlDRESb1T73ju0LgAtqbLJvXUU9akIcTYgBmhFHE2KAZsTRhBigGXE0IQYYYBy1fpEbERHDlWkYIiJaJEk/IqJFkvQjIlpk1i6iImkU+E3gWcBPgBuBr9leP9TAWkjSk4HXs+n7cX5Gb0VsJGkeG/9G7rL9i763Mdu+yJX0h8AJwJ3A1cBDwJOBvYD9qJLNf7d994DjeBXwB1SJ7pl0JDrgX2w/Msj2O+JYQHU9xCYJF7hwEAfVpPbfD/w2cBmbvh+vKdt/afv6QcZRYhnqa1FiaMpxMfQ4mvB+NCEOSTsCxwFvAbYFxqn+LnYHVgOftP2NvrU3C5P+cVTj/38yxf6XAbvYvnSAMVxIdaXxecBaNk10vw38ne2BXpgm6dNUcxt9ZYo4XgEss33FAGM43PaUU6FK2g3Yw/baQcVQ2mnCa9GU42LocTTh/WhKHJIuAc4Avmz7+5P2vQJ4K3CD7dP60t5sS/pNIGlX29POm7EldfoQx4tt3zjN/m2pEu66QcbRBE14LRp0XAw9jia8H02Ko06zLulLmgscC/wO1b9qZuNZzWm2f15jLLtTnUUYuM/2g3W13SWWnQHbfrjmdncETgSOBHajei0eono/lk8+s6kppqG8Fh3tN+K4aFAcQ30/mhBH+Ts5lI73A7hoEH8fszHpnwl8HzidaoI3qCZ2WwLsbPvNNcTwMuAfgR2Bezti+D7wTtvXDDqGEscewN8CB5a2BTwd+DrVv6x31RDDRaW9020/UMqeQfV+HGT74EHHUNpswmvRlONi6HE04f1oShySjgFOAi7mie/HwcAHbJ/R1wZtz6obcOs0+26rKYZrgVd2Kd8XuK7G1+JbwJuBOR1lc6i+tFrdgPdjyn2z9LVoynEx9Dia8H40JQ6q6eN36lI+bxA5azaO039Y0lGSHv/dJG0j6c1AXf+2PdX2lZMLba8GnlpTDAC72v6c7cc6YnjM9lnALjXF8F1J7yldCUDVrSDpvTxxQZ1Ba8Jr0ZTjoglxNOH9aEocourSmewXZV9fzcZx+kcDJwOflDSR5HcCvlH21eFCSedTfSM/kdgWAscAX60pBoCrJX2SqqurM44lwLdriuHNwDLg8jJSB+BBqim131RTDNCM16Ipx0UT4mjC+9GUOD4MXCPp4o4Y9qDq3vlQvxubdX36nSTtQvU71r4SjqTDqNb/nU/1aT0GrHI1y2hdMWxL9aX2JnFQfan907piGbamvBZNOC6aEEeD3o+mxDEPeO2kGC7yAL5UntVJfzJJB9u+ZNhxtJ2kr9s+YNhxRDRJXaOp2pb077a9Rw3tzAHeQfUN/IW2/71j31/b/ptBx1Da2h44nuog+t9UXS2/B3wH+KDtH9UQw+QrbUV14cutALZfMugYShxPGHcu6Q+AfaiuvPwn1/CHUBYQOsv29yQ9F/g08KvAbcA7bN8w6BhKHOcCXwDOq+MYmCKG5wB/TTVa5WTgFOBVwC3Af3NNo3dKLK+h+rtYCGwAbgdOdU1j8yeNphqj+hsZ2GiqWZf0JU11FaGAA2wP/IsqSacC2wNXUV1Nd7ntd5d919jee9AxlLbOpuojfArVIvO3AGdTXXH5DNtvrSGGVcAPgL+hurxdwP8FfgPA9ncHHUOJ4/HXXdJfU11y/69UcwKN2f6LGmK4yfaLyvb5VInli5L2Bz5se79Bx1Davpdq1MoBwNeAM6nmQfpZHe2XGK4o7e5INR3EZ4DPAYcAv1/Xf4KSllNNd3Ap1bUkd1J9CL8T+Ijtz9cQw7XAH0/+cl3SvsD/sf3SvjZYx5CkOm9UI3QOB35r0m1/4MGaYri+Y3su1So45wLbAd+u8bW4tvwU8AAbP+TVGWMNcfwOcAXwhnL/jiEcF9/u2L6GagQLwJOoLnGvI4ZbO7bXTHXM1PVaAE+jOim5gGq+l08Dhwzh/bh7qn01xHFDx/Zc4Jtlex5wY00x3D7NvnX9bm82jt5ZDTxq+/LJOyTdWlMM205s2N4ALJX0P6gu+NihphgeZ9uSLnA5isr92v7Fc3U2ezHwIUnvoOP1qdFTJL2cajrxObZ/XGL7uaTHpn9o35wj6TPAB4EvSvpzqpOBA4GBTgA4ycRx8EPgn4F/LlejvolqpNXFNcTwC0l7UZ3pby9p1PZaSb9CNU6+Lr+QtLOr2XefNdG27Ycl9X245BRqHU0165K+7cOm2ffqmsJYK+lQ24+/YbY/KOk+4FM1xTARxw62f2T77ROFpT/5hzXGQUmy75b0Uqq+27rdD/xd2V4v6Zm27y8jvDbUEYDt96maBfZM4LlU//ktBb4E/H4dMRSb9OOXpPeP5VaH9wBfphqLfiRwYjk2ng78UU0xAHwE+HY5IXw+8KcAkkaA6+oIwPYJU4ym+oQHMJpq1vXpdyPp9ba/Muw4mkSSPKQ3X9L7bb9/GG1PVr503872o8OOpe0k7Qo87I4LpWpqd2fgOVRdKbXPBVW32XhFbjcfHHYAkga2uv3WmIhjWAm/eMMQ235c+fB5bJgJv2nHxZB9pO6ED9V/ObbXTiR8VWtANMIg3pe2JP26+uamMzrsAIomxNGE9wOa8eHThPcDmhFHE2KAmo8LSTtPcdsFeF2/25t1ffpT+ONhB0A1nXATNCGOVww7gKIJHz5NeD+gGXE0IQao/7gYB747qV2X+7t1fUQPZmWfvqTns/FLkYm5qVfZvmWogbWQGrS+QZfYtnFNS/LFL4+6jwtJtwMHussSrpLusb2wn+3Nuu6dMnvjWVSfklcBa8r2mZKW1RTDjpKWS/qOpP9XbreUsp3qiKFBcfwz8DLg/VT/qh4OfAB4KfAvNcWApLmS/ljSVyVdL+k64HxJfyLpSTXF0IT3oxFxNCGGEsfQjwvg76muC+jmb/vd2Kw705d0G/CiyWeQqiZWusn24hpiaMrCIUOPQ9Kttp83xb7bbO816BhKW01YXGfo70dT4mhCDKXNoR8XtavjirM6b1Tzyjy7S/mzqWnRjunaqSuGpsRBdbHcUcA2HWXbUM0DdGVDXou6FtcZ+vvRlDiaEENTjosp2l4xqOeedd07wJ8Dl0q6UNKKcvsq1dwa76ophu+qGQuHNCGOo4E3Ag9Kuq30Xz4I/C71rW8AzVhcpwnvR1PiaEIM0IzjopuBjWSadd07UL1pVDModl7dtsY1jQFWNTf2MqovkycvHHKyq6sfWxNHRzzDXN9gEdVsjgew8Y95HhvXQr2zhhga8X50iWNibqba4mjQa7GIJx4Xolp0qbbjYoq4vmr70IE892xL+tqCWSy3pE70j6R9qK4HWyPphcChwC22LxxSPEP78InmastxMRuT/k+o5sOesgqwo2uYV79r49LbbH+6xvaeT/Ufz2qXScZK+RPmBhpg+ycBh1FdE3IJ8ErgMuAgqpWBPjzoGKaJ7Qzbxwyx/d+gzOlvu45JzibafSXVh+4PJD2F6ox7b+BmqqtiH6khhhOAc22PbbbyYOPYlqqb8V7bl0r6L8CvU01DvsI1DCmWtCNwItUcRCOl+CGqYc3L3eepIWZj0n/2FlR7bFgHm2payKW0dQJwHNUB/DLgXbbPK/tq+W9H0g2l7e2ouhAWdCSbK13fIiqT11kQ8Bqqf+OxPfCrMCVdZXufsv1HVO/NF6nmkP+y7eWDjqG0fRPwUtsbVF3m/yhwDtVsny+1/bs1xPAI8GPgP6gmoPu87fFBt9sljs9SnZBsTzWKZwc2znwq20tqiKHWkUyz7opc17Qox3S06WpRj++iWrChLn8EvML2j0rf5TmSFtn+GPVddbihfJfyqKT/sP0DANs/kVTnhVELqM5kT2Xj1Y6jwP+qMYbOcd9LgYNtj0v6KNUop1qSPtVIqomZRUc7Pvz/TdWCHnW4g+rK7IOoRnJ9QNLVVB8A57qa9rkOv2r7JaouIrwXeJbtxyT9CzXNsgkssn1yZ0FJ/idLevsUj5mxWZf0G2J3qkWOJ3/7L+DfN60+MHNclsOzfZeqFZrOKf8N1ZX0fyZpe1eTmj0+/UL5l7bOpD9KNXrrfVTL8V0r6Sfusu7CAG1TvsDchuoschyqaacl1TK9c3FjRzfjddo4l/1eQF1XSNvVVa8XAxeXC6EOA94CfJSN3RyDtk3p4nkq1dn+jsB6qv9M67o467uS3kN1pv8gMLFe7h8ygJFMSfqD8RVgB9ubnDVJuqzGOB6Q9LKJOMoZ/+uBlVRrs9bh1bZ/WtrvTPJPovr3tRal7VMkfb78fJD6j/8dgaupPnAt6Rm2H5C0A/XO9/IO4GOqlo38HvAtSfdQJZh31BTDE37f0ne+ClhVuv7qchrVtT1zqE4IPi/pDmBfqiv76/Bmqu9VLu8YwjoxmupN/W5s1vXpx0aSFlB1rzzQZd9+tr9ZQwyNHE0l6XBgP9t/VWe7U8SyPbB73cMDJT2Nah75uVTrBD9YY9t72b6trvamI+lZALbvUzUFxEFUSzheNdzIBiNJfwCakuiaEEdTRlM15LUYegxNiaMJMTQsjudSTUq4kGolt9uBMwcxkirdO4Pxgmm+zIWS6FoSx/O3oE4dF8014bVoQgxNiaMJMTQijjLK7reBy4FfA66lSv7fkvRO25f1tb2c6fdfU4aNNiWOJmjCa9GEGJoSRxNiaEocE8Oay6ih7YELbO8vaQ/gPNsv72d7OdMfgCYMG4XmxNEETXgtmhADNCOOJsQAzYmDKhc/RjVq6GkAtu/WAKZ3TtKPiBiuU4E1klYDr6aaCwhJI1TDR/sq3TsREUMm6UXAC6im5PjOQNtK0o+IGJ66RxAl6UdEDFHdw5rTpx8RMVy1DmvOmX5ERIvMxuUSIyJiCkn6EREtkqQfEdEiSfoRES2SpB8xBUlfknS1pJskLS1lx0q6TdJlkv5J0j+U8hFJX5C0ptz2G270Ed1l9E7EFCTtbHt9WdRjDdVqaN+kWkT8h1Trml5n+3hJ/wp80va/lYmyLrL9gqEFHzGFjNOPmNoJkn6nbC8E3gpcbns9QFmFa6+y/yDghdLjC0I9XdLTalzrNWKLJOlHdFHWEz4IeJXtR8syl7dSzY/SzTal7k/qiTBiZtKnH9HdjsDDJeE/n2rN1O2B35I0T9Jc4Pc66l8MHD9xR9LLao02Ygsl6Ud091VgbllV6UPAauBe4CPAlcDXgJuBieXsTgBGJV0v6WbgT+oPOWLz8kVuxFaQtIPtH5Uz/S8CK21/cdhxRWypnOlHbJ33S7oWuBG4E/jSkOOJ2Co504+IaJGc6UdEtEiSfkREiyTpR0S0SJJ+RESLJOlHRLRIkn5ERIv8f3ejStwMILLEAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "df_train.groupby('age').size().plot(kind='bar')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAErCAYAAAAyrlO7AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAauUlEQVR4nO3de5SdVZ3m8e8DERRREkihkkSDGlR0RLEE1LFbRRHUNjgjCu1IdDKm24ambWdGcJxpHJ3uhXNpWpdiTzRpQrcNIoKkBUXEC8tZAgkoyJ0MICmDpOxEdMQb8Mwf7y7PoXLqepLzFuzns1atOme/+9T51U7ynJ393mSbiIiow25tFxAREYOT0I+IqEhCPyKiIgn9iIiKJPQjIiqS0I+IqMi8qTpIWgu8Cdhq+wVd7X8KnAw8CFxi+wOl/YPASuAh4BTbl5X2o4GPA7sDn7V9xlTvvXDhQi9dunSmv1NERNWuvfban9ge6rVtytAHzgY+CZwz1iDp1cBy4IW2fy1p/9J+MHA88HzgAODrkg4qL/sU8DpgBNggab3tmyd746VLl7Jx48ZplBgREWMk/XCibVOGvu0rJS0d1/xe4Azbvy59tpb25cB5pf0uSZuAw8q2TbbvLAWdV/pOGvoREbFzzXZN/yDglZKulvRtSS8t7YuAzV39RkrbRO07kLRK0kZJG0dHR2dZXkRE9DLb0J8HLACOAP4jcL4kAerR15O079hor7Y9bHt4aKjnklRERMzSdNb0exkBLnRz4Z5rJD0MLCztS7r6LQa2lMcTtUdExIDMdqb/JeA1AGVH7R7AT4D1wPGS9pR0ILAMuAbYACyTdKCkPWh29q7vt/iIiJiZ6RyyeS7wKmChpBHgdGAtsFbSjcBvgBVl1n+TpPNpdtA+CJxk+6Hyc04GLqM5ZHOt7Zt2we8TERGT0Fy+tPLw8LBzyGZExMxIutb2cK9tOSM3IqIis92RGxGPEUtPu6TtEgC4+4w3tl1CFTLTj4ioSEI/IqIiCf2IiIok9CMiKpLQj4ioSEI/IqIiCf2IiIok9CMiKpLQj4ioSEI/IqIiCf2IiIok9CMiKpLQj4ioSEI/IqIiCf2IiIok9CMiKjJl6EtaK2lruR/u+G3/QZIlLSzPJekTkjZJukHSoV19V0i6o3yt2Lm/RkRETMd0ZvpnA0ePb5S0BHgdcE9X8zHAsvK1Cvh06bsvzQ3VDwcOA06XtKCfwiMiYuamDH3bVwLbemw6E/gA0H1n9eXAOW5cBcyX9DTg9cDltrfZ3g5cTo8PkoiI2LVmtaYv6c3Aj2xfP27TImBz1/OR0jZRe6+fvUrSRkkbR0dHZ1NeRERMYMahL2kv4EPAX/Ta3KPNk7Tv2Givtj1se3hoaGim5UVExCRmM9N/FnAgcL2ku4HFwHWSnkozg1/S1XcxsGWS9oiIGKAZh77tH9je3/ZS20tpAv1Q2z8G1gMnlqN4jgDut30vcBlwlKQFZQfuUaUtIiIGaDqHbJ4LfBd4jqQRSSsn6X4pcCewCfgM8CcAtrcBHwU2lK+PlLaIiBigeVN1sH3CFNuXdj02cNIE/dYCa2dYX0RE7EQ5IzcioiIJ/YiIiiT0IyIqMuWafsRj0dLTLmm7BADuPuONbZcQlclMPyKiIgn9iIiKJPQjIiqS0I+IqEhCPyKiIgn9iIiKJPQjIiqS0I+IqEhCPyKiIgn9iIiKJPQjIiqS0I+IqEhCPyKiIgn9iIiKTOceuWslbZV0Y1fb/5B0q6QbJF0kaX7Xtg9K2iTpNkmv72o/urRtknTazv9VIiJiKtOZ6Z8NHD2u7XLgBbZfCNwOfBBA0sHA8cDzy2vOkrS7pN2BTwHHAAcDJ5S+ERExQFOGvu0rgW3j2r5m+8Hy9CpgcXm8HDjP9q9t3wVsAg4rX5ts32n7N8B5pW9ERAzQzljT/7fAV8rjRcDmrm0jpW2i9h1IWiVpo6SNo6OjO6G8iIgY01foS/oQ8CDwubGmHt08SfuOjfZq28O2h4eGhvopLyIixpn1PXIlrQDeBBxpeyzAR4AlXd0WA1vK44naIyJiQGY105d0NHAq8GbbD3RtWg8cL2lPSQcCy4BrgA3AMkkHStqDZmfv+v5Kj4iImZpypi/pXOBVwEJJI8DpNEfr7AlcLgngKtt/bPsmSecDN9Ms+5xk+6Hyc04GLgN2B9bavmkX/D4RETGJKUPf9gk9mtdM0v8vgb/s0X4pcOmMqouIiJ0qZ+RGRFQkoR8RUZGEfkRERWZ9yGZExGPN0tMuabsEAO4+44277Gdnph8RUZGEfkRERRL6EREVSehHRFQkoR8RUZGEfkRERRL6EREVSehHRFQkoR8RUZGEfkRERRL6EREVSehHRFQkoR8RUZHp3C5xLc0N0LfafkFp2xf4PLAUuBt4m+3tau6d+HHgDcADwLtsX1deswL4z+XH/jfb63burxJTqeEKghExuenM9M8Gjh7Xdhpwhe1lwBXlOcAxNDdDXwasAj4Nv/uQOB04HDgMOF3Sgn6Lj4iImZky9G1fCWwb17wcGJuprwOO7Wo/x42rgPmSnga8Hrjc9jbb24HL2fGDJCIidrHZruk/xfa9AOX7/qV9EbC5q99IaZuofQeSVknaKGnj6OjoLMuLiIhedvaOXPVo8yTtOzbaq20P2x4eGhraqcVFRNRutqF/X1m2oXzfWtpHgCVd/RYDWyZpj4iIAZpt6K8HVpTHK4CLu9pPVOMI4P6y/HMZcJSkBWUH7lGlLSIiBmg6h2yeC7wKWChphOYonDOA8yWtBO4BjivdL6U5XHMTzSGb7wawvU3SR4ENpd9HbI/fORwREbvYlKFv+4QJNh3Zo6+Bkyb4OWuBtTOqbifIsekRER05IzcioiIJ/YiIiiT0IyIqktCPiKhIQj8ioiIJ/YiIiiT0IyIqktCPiKhIQj8ioiIJ/YiIiiT0IyIqktCPiKhIQj8ioiIJ/YiIiiT0IyIqktCPiKhIQj8ioiIJ/YiIivQV+pL+XNJNkm6UdK6kx0s6UNLVku6Q9HlJe5S+e5bnm8r2pTvjF4iIiOmbdehLWgScAgzbfgGwO3A88DHgTNvLgO3AyvKSlcB2288Gziz9IiJigPpd3pkHPEHSPGAv4F7gNcAFZfs64NjyeHl5Ttl+pCT1+f4RETEDsw592z8C/idwD03Y3w9cC/zU9oOl2wiwqDxeBGwur32w9N9v/M+VtErSRkkbR0dHZ1teRET00M/yzgKa2fuBwAHAE4FjenT12Esm2dZpsFfbHrY9PDQ0NNvyIiKih36Wd14L3GV71PZvgQuBlwPzy3IPwGJgS3k8AiwBKNv3Abb18f4RETFD/YT+PcARkvYqa/NHAjcD3wTeWvqsAC4uj9eX55Tt37C9w0w/IiJ2nX7W9K+m2SF7HfCD8rNWA6cC75e0iWbNfk15yRpgv9L+fuC0PuqOiIhZmDd1l4nZPh04fVzzncBhPfr+Cjiun/eLiIj+5IzciIiKJPQjIiqS0I+IqEhCPyKiIgn9iIiKJPQjIiqS0I+IqEhCPyKiIgn9iIiKJPQjIiqS0I+IqEhCPyKiIgn9iIiKJPQjIiqS0I+IqEhCPyKiIgn9iIiKJPQjIirSV+hLmi/pAkm3SrpF0ssk7Svpckl3lO8LSl9J+oSkTZJukHTozvkVIiJiuvqd6X8c+Krt5wKHALfQ3PD8CtvLgCvo3AD9GGBZ+VoFfLrP946IiBmadehLejLwe8AaANu/sf1TYDmwrnRbBxxbHi8HznHjKmC+pKfNuvKIiJixfmb6zwRGgb+T9D1Jn5X0ROAptu8FKN/3L/0XAZu7Xj9S2h5B0ipJGyVtHB0d7aO8iIgYr5/QnwccCnza9ouBX9BZyulFPdq8Q4O92vaw7eGhoaE+youIiPH6Cf0RYMT21eX5BTQfAveNLduU71u7+i/pev1iYEsf7x8RETM069C3/WNgs6TnlKYjgZuB9cCK0rYCuLg8Xg+cWI7iOQK4f2wZKCIiBmNen6//U+BzkvYA7gTeTfNBcr6klcA9wHGl76XAG4BNwAOlb0REDFBfoW/7+8Bwj01H9uhr4KR+3i8iIvqTM3IjIiqS0I+IqEhCPyKiIgn9iIiKJPQjIiqS0I+IqEhCPyKiIgn9iIiKJPQjIiqS0I+IqEhCPyKiIgn9iIiKJPQjIiqS0I+IqEhCPyKiIgn9iIiKJPQjIiqS0I+IqEjfoS9pd0nfk/Tl8vxASVdLukPS58v9c5G0Z3m+qWxf2u97R0TEzOyMmf6fAbd0Pf8YcKbtZcB2YGVpXwlst/1s4MzSLyIiBqiv0Je0GHgj8NnyXMBrgAtKl3XAseXx8vKcsv3I0j8iIgak35n+3wAfAB4uz/cDfmr7wfJ8BFhUHi8CNgOU7feX/o8gaZWkjZI2jo6O9lleRER0m3XoS3oTsNX2td3NPbp6Gts6DfZq28O2h4eGhmZbXkRE9DCvj9e+AnizpDcAjweeTDPzny9pXpnNLwa2lP4jwBJgRNI8YB9gWx/vHxERMzTrmb7tD9pebHspcDzwDdvvAL4JvLV0WwFcXB6vL88p279he4eZfkRE7Dq74jj9U4H3S9pEs2a/prSvAfYr7e8HTtsF7x0REZPoZ3nnd2x/C/hWeXwncFiPPr8CjtsZ7xcREbOTM3IjIiqS0I+IqEhCPyKiIgn9iIiKJPQjIiqS0I+IqEhCPyKiIgn9iIiKJPQjIiqS0I+IqEhCPyKiIgn9iIiKJPQjIiqS0I+IqEhCPyKiIgn9iIiKJPQjIioy69CXtETSNyXdIukmSX9W2veVdLmkO8r3BaVdkj4haZOkGyQdurN+iYiImJ5+ZvoPAv/e9vOAI4CTJB1Mc+/bK2wvA66gcy/cY4Bl5WsV8Ok+3jsiImZh1qFv+17b15XHPwduARYBy4F1pds64NjyeDlwjhtXAfMlPW3WlUdExIztlDV9SUuBFwNXA0+xfS80HwzA/qXbImBz18tGStv4n7VK0kZJG0dHR3dGeRERUfQd+pL2Br4IvM/2zybr2qPNOzTYq20P2x4eGhrqt7yIiOjSV+hLehxN4H/O9oWl+b6xZZvyfWtpHwGWdL18MbCln/ePiIiZ6efoHQFrgFts/3XXpvXAivJ4BXBxV/uJ5SieI4D7x5aBIiJiMOb18dpXAO8EfiDp+6XtPwFnAOdLWgncAxxXtl0KvAHYBDwAvLuP946IiFmYdejb/g691+kBjuzR38BJs32/iIjoX87IjYioSEI/IqIiCf2IiIok9CMiKpLQj4ioSEI/IqIiCf2IiIok9CMiKpLQj4ioSEI/IqIiCf2IiIok9CMiKpLQj4ioSEI/IqIiCf2IiIok9CMiKpLQj4ioSEI/IqIiAw99SUdLuk3SJkmnDfr9IyJqNtDQl7Q78CngGOBg4ARJBw+yhoiImg16pn8YsMn2nbZ/A5wHLB9wDRER1ZLtwb2Z9FbgaNv/rjx/J3C47ZO7+qwCVpWnzwFuG1iBE1sI/KTtIuaIjEVHxqIjY9ExF8biGbaHem2YN+BC1KPtEZ86tlcDqwdTzvRI2mh7uO065oKMRUfGoiNj0THXx2LQyzsjwJKu54uBLQOuISKiWoMO/Q3AMkkHStoDOB5YP+AaIiKqNdDlHdsPSjoZuAzYHVhr+6ZB1jBLc2q5qWUZi46MRUfGomNOj8VAd+RGRES7ckZuRERFEvoRERVJ6EdEVCShHxFRkUGfnPWoImkYeCVwAPBL4Ebg67a3tVpYRMQs5eidHiS9CzgFuAu4FtgKPB44CHgFTfj/F9v3tFXjIEl6GfBvaD4An0bnA/AS4B9s399ieQOVseiQtJjmXJvxE6NLgK/YfrjF8gZG0s+m6gLca/ugQdQzlYR+D5JOojmH4JcTbH8RsJ/tKwZb2eBJ+grNWdMXAxt55Afgq4E/AP7a9mP+JLuMRYekvwMWAV+m91i8BDjN9pWtFTkgkr5n+8X99hmUhH5MStJC25NePGo6fR4LMhYdkl5g+8ZJtu8BPN32pgGW1QpJz7R9Z799BiWh34OkecBK4C00/201nRneGtu/bbG8Vkh6Cs3MzsAW2/e1XFJrMhaPJGlfwLa3t13LXCDpicCvbD/Udi29JPR7kHQu8FNgHc1F4qC5ONwKYF/bb2+rtkErS1l/C+wD/Kg0L6YZnz+xfV1btQ1axqJD0tOB/w4cSfP7C3gy8A2aZZ2726tusCTtRrNv4x3AS4FfA3sCo8ClwGrbd7RX4SMl9HuQdJvt50yw7fa5skNmECR9H/gj21ePaz8C+N+2D2mnssHLWHRI+i7wN8AFYzPacme844D32T6izfoGSdK3ga/TrATcOLYDu/wP6NXAHwIX2f6H9qrsSOj3IOkq4H8BX+z6A9yN5i/0+20f3mZ9gyTpDtvLJti2yfazB11TWzIWHVOMxYTbHoskPW6qJd/p9BmUHKff2/HAx4CzJG2n+a/rPsA3y7aafEXSJcA5wObStgQ4Efhqa1W1I2PRca2ks2iWQLvHYgXwvdaqaoHt30oSze1gf7evB7jGZVY9VwIfMtOfkqT9aMbpMX9ExkQkHUNzL+NFNB+AI8B625e2WlgLMhaNcnTOSnqMBc3BDr9usbyBknQUcBZwB4/c1/Nsmn09X2urtl4S+tMg6UDgxcDNtm9tu56ImDsk3QIcM37ndcmNS20/r5XCJpBr7/Qg6Utdj5fTHJHwB8D6crZuNSSdLGlhefwsSVdK2i7pakn/ou36BknShZLeIWnvtmtpm6RnSlor6aOS9pb0GUk3SvqCpKVt1zdg8+gc5dftR8DjBlzLlBL6vT2j6/GpwGtsvxt4OfDn7ZTUmvd2LW19AjjT9gKacfnb9spqxeE0527cI+l8SW8pyxw1Opvm9qe/AK4CbgOOodm3sba9slqxFtgg6VRJf1i+TgWuBta0XNsOsrzTg6TrbB9aHl9j+7CubXPmdOpB6D58VdIG2y/t2naD7Re2V91gjf3ZS3oScCxwAs1x2V8Gzp1ra7e7Uve/A0n32H56r221kHQw8GZ23Ndzc6uF9ZCjd3o7pFxEScCekp5q+8dlVrd7y7UN2gWSzgY+Alwk6X3AhTQn5VRxwbkuY0di/Bz4e+Dvy7HYbwNOA6oJfeBhSQfRHNW2l6Rh2xslPZv6/o1Qwv3mR8PZyZnpz4Ck+cDzbH+37VoGqezHeC/wLJozDTcDXwI+VtlVJa+0/Xtt1zEXSDqS5oiVh4H30Cx7HkJzVu57bF/cYnkD1XV28muAsX8P+zBHz05O6E+TpDfZ/nLbdUTMVWWH//a5es2ZXeXRdnZyduRO30faLmCukLS67RrmiozFI/xVbYFfLLT9+e7f3fZDts8D9muxrp6ypj99aruAOWS47QLmkIxFR61j8ag6OzmhP31/1HYBc8jWtguYQzIWHbWOxYk0Zyf/VzpH72wG/okcsvnoIem5dE4xH7uWxnrbt7RaWEREH7Km30M5seI8mk/sa2hOQhFwrqTT2qxt0CTtI+kMSbdK+ufydUtpm992fYOUsejIWEyPpL9ou4bxMtPvQdLtwPPHXxmvHKd/U2WXjb2M5tCzdbZ/XNqeSrNe+Vrbr2uzvkHKWHRkLKZn/Ilrc0FCvwdJtwKvt/3Dce3PAL420Q1WHos0+Q1lJtz2WJSx6MhYdJQTOXtuAp5ge07tO51Txcwh7wOukHQHnb3xT6e5VOrJrVXVjh9K+gDNjO4++N09Yt9FZ2xqkbHoyFh0/BR4aa97JUuac2OR0O/B9lfLKeZjN0UYu5bGhgqPQ347zSUGvi1p/9J2H81109/WWlXtGD8WAn5MxqL2vxfn0FykcYfQB/5xwLVMKcs7PXRfcK2fPhERc01CvwdJv6S5C86EXYB95toOmkGQ9C9p/gd0Y01XlQSQdDhwi+2fSXoCzUz3UOBmmrNRa7oO0SnAhbZ7XUe+epI+bPvDbdfRS0K/h7LDdioP1fAXvvvS0pLeA5wEXAQcBfyT7TParG+QJN0EHGL7wXL5hQeAC2iuOHqI7X/VaoEDJOl+mmvp/1/gXOALtkfbrWrumMsrAQn9mNS466ZvAN5ge1TSE4GrbFdz9yxJt4zd+m78P2pJ37f9ovaqGyxJ3wNeAryWZn3/zcC1NB8AF5bLT1drLt9TICdnxVR2k7RAnRvEjwLY/gXwYLulDdyNkt5dHl8vaRig7PT/7cQve0yy7Ydtf832SuAAmkstHw3c2W5pc8JL2i5gIpnpx6Qk3U1zzXTRXI7i5eWGMnsD36lsdrsP8HHglcBPaNbzN5evU2xf32J5AzXZTFbSE2z/ctA1tUXSPJpr77yF5sNv7LItFwNrxp/k2baEfsyKpL2Ap9i+q+1aBq3cLvGZlBti9zo++7FO0kG2b2+7jrlA0rk0x+qvo3OD9MU0Zyfva/vtbdXWS0I/JpXDVzsyFh0Zi44pzk6+3fZBg65pMjk5K6byPEk3TLJdNLeGq0HGoiNj0bFd0nHAF20/DCBpN5o7Z825e+Um9GMqz51Gn1rOUs5YdGQsOo4HPgacJWk7zQfefJoL0h3fZmG9ZHknImIn6TrK7Sdt1zKRHLIZEdEHSXtIOlHSkbb/GThK0iclnSTpcW3XN15m+hERfZD0OZql8r1ojuLZG7iQ5kxt2V7RYnk7SOhHRPRB0g22X1iO1/8RcIDthyQJuN72C1su8RGyvBMR0Z/dyl31nkQz2x87amlPYM4t7+TonYiI/qwBbgV2Bz4EfEHSncARNPfanlOyvBMR0SdJBwDY3lJuDP9a4B7b17Rb2Y4S+hERfXi0nZ2c0I+I6MOj7aZLWdOPiOjPo+rs5Mz0IyIqkkM2IyIqktCPiKhIQj8ioiIJ/YiIiiT0IyYg6UuSrpV0k6RVpW2lpNslfUvSZyR9srQPSfqipA3l6xXtVh/RW47eiZiApH1tb5P0BGAD8Hrg/9DcEP3nNDfJuN72yZL+ETjL9nckPR24zPbzWis+YgI5Tj9iYqdIekt5vAR4J/Bt29sAJH0BGLv/6WuBg5sLKwLwZElPsv3zQRYcMZWEfkQPkl5FE+Qvs/2ApG8BtwETzd53K31/OZgKI2Yna/oRve0DbC+B/1yaKybuBfy+pAXl2un/uqv/14CTx55IetFAq42YpoR+RG9fBeZJugH4KHAVzQ0y/gq4Gvg6cDNwf+l/CjAs6QZJNwN/PPiSI6aWHbkRMyBpb9v/r8z0LwLW2r6o7boipisz/YiZ+bCk7wM3AncBX2q5nogZyUw/IqIimelHRFQkoR8RUZGEfkRERRL6EREVSehHRFTk/wPwLvUqkG7oKgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "def normalize_age(df):    \n",
    "    df['age'] = df['age']. \\\n",
    "                      map(lambda x: '[0-50)' if x in ['[0-10)', '[10-20)', '[20-30)', '[30-40)', '[40-50)'] else x). \\\n",
    "                      map(lambda x: '[80-100)]' if x in ['[80-90)', '[90-100)'] else x)\n",
    "    return df\n",
    "\n",
    "df_train = normalize_age(df_train)\n",
    "df_test = normalize_age(df_test)\n",
    "df_val = normalize_age(df_val)\n",
    "df_train.groupby('age').size().plot(kind='bar')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Feature engineering: Medicine columns\n",
    "By looking at the statistics we can see that most of these columns are almost constant, so we decided to remove them.\n",
    "\n",
    "The only relevant column is 'insulin'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Column   # dominant entry   % dominant entry  total entries\n",
      "metformin  4793  79%  6000\n",
      "repaglinide  5923  98%  6000\n",
      "nateglinide  5964  99%  6000\n",
      "chlorpropamide  5992  99%  6000\n",
      "glimepiride  5710  95%  6000\n",
      "acetohexamide  6000  100%  6000\n",
      "glipizide  5221  87%  6000\n",
      "glyburide  5297  88%  6000\n",
      "tolbutamide  5999  99%  6000\n",
      "pioglitazone  5605  93%  6000\n",
      "rosiglitazone  5545  92%  6000\n",
      "acarbose  5985  99%  6000\n",
      "miglitol  5996  99%  6000\n",
      "troglitazone  6000  100%  6000\n",
      "tolazamide  5998  99%  6000\n",
      "examide  6000  100%  6000\n",
      "citoglipton  6000  100%  6000\n",
      "insulin  3116  51%  6000\n",
      "glyburide.metformin  5968  99%  6000\n",
      "glipizide.metformin  6000  100%  6000\n",
      "glimepiride.pioglitazone  6000  100%  6000\n",
      "metformin.rosiglitazone  6000  100%  6000\n",
      "metformin.pioglitazone  6000  100%  6000\n"
     ]
    }
   ],
   "source": [
    "total = df_train.shape[0]\n",
    "medication_columns = ['metformin', 'repaglinide', 'nateglinide', 'chlorpropamide', 'glimepiride', 'acetohexamide', 'glipizide', 'glyburide', 'tolbutamide',\n",
    "                      'pioglitazone', 'rosiglitazone', 'acarbose', 'miglitol', 'troglitazone', 'tolazamide', 'examide', \n",
    "                      'citoglipton', 'insulin', 'glyburide.metformin', 'glipizide.metformin', 'glimepiride.pioglitazone', \n",
    "                      'metformin.rosiglitazone', 'metformin.pioglitazone']\n",
    "\n",
    "print('Column   # dominant entry   % dominant entry  total entries')\n",
    "for column in medication_columns:\n",
    "    dominant_cnt = df_train[column].value_counts()[0]\n",
    "    percentage_dominant_cnt = int(dominant_cnt / total * 100)\n",
    "    print(\"{}  {}  {}%  {}\".format(column, dominant_cnt, str(percentage_dominant_cnt), str(total)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "No        3116\n",
       "Steady    1794\n",
       "Down       590\n",
       "Up         500\n",
       "Name: insulin, dtype: int64"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train['insulin'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "#medication_columns.remove('insulin')\n",
    "\n",
    "df_train.drop(medication_columns, axis='columns', inplace=True)\n",
    "df_val.drop(medication_columns, axis='columns', inplace=True)\n",
    "df_test.drop(medication_columns, axis='columns', inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['race', 'gender', 'age', 'admission_type_id',\n",
       "       'discharge_disposition_id', 'admission_source_id', 'time_in_hospital',\n",
       "       'num_lab_procedures', 'num_procedures', 'num_medications',\n",
       "       'number_outpatient', 'number_emergency', 'number_inpatient', 'diag_1',\n",
       "       'diag_2', 'diag_3', 'number_diagnoses', 'max_glu_serum', 'A1Cresult',\n",
       "       'change', 'diabetesMed', 'readmitted', 'diag_1_desc', 'diag_2_desc',\n",
       "       'diag_3_desc'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.columns"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Feature engineering: discharge, admission type and admission source\n",
    "We reduce the dimensionality by grouping disposition and admission into fewer categories.\n",
    "\n",
    "Discharge disposition split into those sent home or not.\n",
    "\n",
    "Admission source split into Emergency room, Physician Referral and Other.\n",
    "\n",
    "Admission type split into Emergency and Other."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "for dataset in [df_train, df_val, df_test]:\n",
    "    dataset['discharge_disposition_id'].replace('Not Mapped', '?', inplace=True)\n",
    "    dataset['admission_source_id'].replace(['Not Available', 'Not Mapped'], ['?', '?'], inplace=True)\n",
    "    dataset['admission_type_id'].replace(['Not Available', 'Not Mapped'], ['?', '?'], inplace=True)\n",
    "    \n",
    "    home_vals = ['Discharged to home', \\\n",
    "                    'Discharged/transferred to home with home health service', \\\n",
    "                    'Discharged/transferred to home under care of Home IV provider']\n",
    "    safe_vals = ['Emergency Room', 'Physician Referral']\n",
    "    dataset['discharge_disposition_id'] = pd.Series(['Home' if val in home_vals else 'Not Home' for val in dataset['discharge_disposition_id']], index=dataset.index)\n",
    "    dataset['admission_source_id'] = pd.Series([val if val in safe_vals else 'Other' for val in dataset['admission_source_id']], index=dataset.index)\n",
    "    dataset['admission_type_id'] = pd.Series([val if val == 'Emergency' else 'Other' for val in dataset['admission_type_id']], index=dataset.index)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Feature engineering: diag\n",
    "\n",
    "For the diagnosis text, there is a one to one mapping between diagnostic codes and descriptions. Thus, we do not attempt to learn word embeddings in order to find some vector representation for each diagnosis text.\n",
    "\n",
    "However, similar text in the diagnosis description of two diagnoses may indicate some similarity between them. To encode this, e take vectors of frequency counts of words that appear with a frequency of at least 5% the size of the corpus, ignoring stopwords, as a representation of that text. We choose to avoid TF-idf due to the repetition of the same diagnosis texts.\n",
    "\n",
    "For the diagnostic codes, we take two different approaches:\n",
    "\n",
    "- For the first one, we attempt to drastically reduce the dimensionality of the data by keeping only the primary diagnosis and splitting these into those with a diabetes diagnosis (Codes 250) and those without. \n",
    "\n",
    "- For the second approach, we combine all diagnosis codes and represent a patient diagnosis as a vector whose length is the number of different diagnosis codes. In order to prevent a huge blow up in dimensions, we take only the top 200 most frequent diagnoses, and group all others under 'uncommon'. There is a 1 for each of the diagnoses for that patient and a 0 everywhere else. \n",
    "\n",
    "We treat the diagnosis text the same way in both.\n",
    "\n",
    "Despite the one-to-one mapping between diagnosis text and codes, we find that incorporating the text results in a slight improvement over not having it. We speculate that the text therefore encodes some meaningful information regarding the similarity of diagnoses."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train_alt = df_train.copy()\n",
    "df_test_alt = df_test.copy()\n",
    "df_val_alt = df_val.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[True, True, True]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[(df_train['diag_' + str(i)].value_counts().values == df_train['diag_' + str(i) + '_desc'].value_counts().values).all() for i in [1, 2, 3]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "for dataset in [df_train, df_val, df_test]:\n",
    "    dataset['diag_1'] = pd.Series([\"yes\" if val[0:4] == '250' else \"no\" for val in dataset['diag_1']], index=dataset.index)\n",
    "    dataset['diag_desc'] = dataset[['diag_1_desc', 'diag_2_desc', 'diag_3_desc']].agg(' '.join, axis=1)\n",
    "    dataset.drop(columns=['diag_2', 'diag_3', 'diag_2_desc', 'diag_3_desc'], inplace=True)\n",
    "    \n",
    "vectorizer = CountVectorizer(min_df=0.05, stop_words='english')\n",
    "vectorizer.fit(df_train.diag_desc.append([df_val.diag_desc, df_test.diag_desc]))\n",
    "vocab = list(vectorizer.vocabulary_.keys())\n",
    "\n",
    "for dataset in [df_train, df_val, df_test]:\n",
    "    dataset[vocab] = pd.DataFrame(vectorizer.transform(dataset.diag_desc).todense(), index=dataset.index)\n",
    "    dataset.drop(columns=['diag_desc'], inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train.to_csv(os.path.join('data', 'diab_train_filtered.csv'))\n",
    "df_test.to_csv(os.path.join('data', 'diab_test_filtered.csv'))\n",
    "df_val.to_csv(os.path.join('data', 'diab_validation_filtered.csv'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Alternatively, we keep all the diagnosis information and create one-hot encodings for each patient\n",
    "We combine diag_1, 2 and 3 into a one hot encoding for each patient combining the 3.\n",
    "\n",
    "Take only the 200 most frequent diagnoses, grouping the rest into a new one called 'uncommon'."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "l = df_train_alt[['diag_1', 'diag_2', 'diag_3']].values.tolist()\n",
    "l = [list(set(li)) for li in l]\n",
    "l = [el for li in l for el in li]\n",
    "uncommon = [el[0] for el in Counter(l).most_common()[201:]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "for dataset in [df_train_alt, df_val_alt, df_test_alt]:\n",
    "    dataset['diag_1'].replace(uncommon, 'uncommon', inplace=True)\n",
    "    dataset['diag_2'].replace(uncommon, 'uncommon', inplace=True)\n",
    "    dataset['diag_3'].replace(uncommon, 'uncommon', inplace=True)\n",
    "    dataset['diag_desc'] = dataset[['diag_1_desc', 'diag_2_desc', 'diag_3_desc']].agg(' '.join, axis=1)\n",
    "    dataset.drop(columns=['diag_2_desc', 'diag_3_desc'], inplace=True)\n",
    "    dataset['diag_full'] = dataset[['diag_1', 'diag_2', 'diag_3']].values.tolist()\n",
    "\n",
    "mlb = MultiLabelBinarizer()\n",
    "# avoid oob error, capture all possible diagnosis codes\n",
    "mlb.fit(df_train_alt.diag_full.append([df_val_alt.diag_full, df_test_alt.diag_full]))\n",
    "for dataset in [df_train_alt, df_val_alt, df_test_alt]:\n",
    "    dataset[mlb.classes_] = pd.DataFrame(mlb.transform(dataset.diag_full), index=dataset.index)\n",
    "    dataset.drop(columns=['diag_1', 'diag_2', 'diag_3', 'diag_full'], inplace=True)\n",
    "\n",
    "vectorizer = CountVectorizer(min_df=0.05, stop_words='english')\n",
    "vectorizer.fit(df_train_alt.diag_desc.append([df_val_alt.diag_desc, df_test_alt.diag_desc]))\n",
    "vocab = list(vectorizer.vocabulary_.keys())\n",
    "\n",
    "for dataset in [df_train_alt, df_val_alt, df_test_alt]:\n",
    "    dataset[vocab] = pd.DataFrame(vectorizer.transform(dataset.diag_desc).todense(), index=dataset.index)\n",
    "    dataset.drop(columns=['diag_desc'], inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train_alt.to_csv(os.path.join('data', 'diab_train_alt.csv'))\n",
    "df_test_alt.to_csv(os.path.join('data', 'diab_test_alt.csv'))\n",
    "df_val_alt.to_csv(os.path.join('data', 'diab_validation_alt.csv'))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}