credit_card_fraud_detection.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**FINANCIAL FRAUD DETECTION**\n",
    "===="
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Part 1: Reading the input data file and storing to DataFrame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "from pyspark.sql.types import *\n",
    "import sys\n",
    "import os\n",
    "sqlContext = SQLContext(sc)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Read input file and convert it to RDD.\n",
    "1. Read from file system\n",
    "2. split columns\n",
    "3. filter header\n",
    "4. filter empty rows"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Set up AWS S3 access credentials\n",
    "ACCESS_KEY = \"<ACCESS_KEY>\"\n",
    "SECRET_KEY = \"<SECRET_KEY>\"\n",
    "ENCODED_SECRET_KEY = SECRET_KEY.replace(\"/\", \"%2F\")\n",
    "AWS_BUCKET_NAME = \"<BUCKET_NAME>\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "filename = 'PS_20174392719_1491204439457_log.csv'\n",
    "\n",
    "def convertToTransactionSchema(arr):\n",
    "    res = arr\n",
    "    res[0] = int(arr[0]) # step\n",
    "    res[2] = float(arr[2]) # amount\n",
    "    res[4] = float(arr[4]) # old balance\n",
    "    res[5] = float(arr[5]) # new balance\n",
    "    res[7] = float(arr[7]) # old balance destination\n",
    "    res[8] = float(arr[8]) # new balance destination\n",
    "    res[9] = int(arr[9]) # is fraud\n",
    "    res[10] = int(arr[10]) # is flagged fraud\n",
    "    return res\n",
    "\n",
    "transactionsRDD = (sc.textFile(\"s3n://%s:%s@%s/%s\" % (ACCESS_KEY, SECRET_KEY, AWS_BUCKET_NAME, filename))\n",
    "       .map(lambda line: line.split(\",\"))\n",
    "       .filter(lambda line: line[0] != \"step\")\n",
    "       .filter(lambda line: len(line)>1)\n",
    "       .map(convertToTransactionSchema))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "source": [
    "Convert transactionsRDD into DataFrame and cache it."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>.container { width:100% !important; }</style>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+----+-------+-------+-----------+--------------+--------------+-----------+--------------+--------------+-------+--------------+\n",
      "|step|   type| amount|   nameOrig|oldbalanceOrig|newbalanceOrig|   nameDest|oldbalanceDest|newbalanceDest|isFraud|isFlaggedFraud|\n",
      "+----+-------+-------+-----------+--------------+--------------+-----------+--------------+--------------+-------+--------------+\n",
      "|   1|PAYMENT|9839.64|C1231006815|      170136.0|     160296.36|M1979787155|           0.0|           0.0|      0|             0|\n",
      "|   1|PAYMENT|1864.28|C1666544295|       21249.0|      19384.72|M2044282225|           0.0|           0.0|      0|             0|\n",
      "+----+-------+-------+-----------+--------------+--------------+-----------+--------------+--------------+-------+--------------+\n",
      "only showing top 2 rows\n",
      "\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "transactionSchema = StructType([\n",
    "    StructField(\"step\", IntegerType(), True),\n",
    "    StructField(\"type\", StringType(), True),\n",
    "    StructField(\"amount\", FloatType(), True),\n",
    "    StructField(\"nameOrig\", StringType(), True),\n",
    "    StructField(\"oldbalanceOrig\", FloatType(), True),\n",
    "    StructField(\"newbalanceOrig\", FloatType(), True),\n",
    "    StructField(\"nameDest\", StringType(), True),\n",
    "    StructField(\"oldbalanceDest\", FloatType(), True),\n",
    "    StructField(\"newbalanceDest\", FloatType(), True),\n",
    "    StructField(\"isFraud\", IntegerType(), True),\n",
    "    StructField(\"isFlaggedFraud\", IntegerType(), True)])\n",
    "\n",
    "transactionsDF = sqlContext.createDataFrame(transactionsRDD, transactionSchema)\n",
    "transactionsDF.cache()\n",
    "\n",
    "# Improve the display format\n",
    "from IPython.core.display import display, HTML\n",
    "display(HTML(\"<style>.container { width:100% !important; }</style>\"))\n",
    "\n",
    "print transactionsDF.show(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Check if the DataFrame is cached."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "transactionsDF.is_cached"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Part 2: Data Cleaning"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The purpose of data cleaning is to check if there is any invalid or missing values, and eliminate those reords."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Print the datatype for each column."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- step: integer (nullable = true)\n",
      " |-- type: string (nullable = true)\n",
      " |-- amount: float (nullable = true)\n",
      " |-- nameOrig: string (nullable = true)\n",
      " |-- oldbalanceOrig: float (nullable = true)\n",
      " |-- newbalanceOrig: float (nullable = true)\n",
      " |-- nameDest: string (nullable = true)\n",
      " |-- oldbalanceDest: float (nullable = true)\n",
      " |-- newbalanceDest: float (nullable = true)\n",
      " |-- isFraud: integer (nullable = true)\n",
      " |-- isFlaggedFraud: integer (nullable = true)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "transactionsDF.printSchema()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We need to check each column for invalid values. The set of invalid values depends on the data type of the columns:\n",
    "1. For columns with string type: empty or null values.\n",
    "2. Otherwise: null or minus values.\n",
    "\n",
    "Firstly, we need to get columns that has 'string' datatype and store it in a variable."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['type', 'nameOrig', 'nameDest']\n"
     ]
    }
   ],
   "source": [
    "stringCols = []\n",
    "for (colName, colType) in transactionsDF.dtypes:\n",
    "    if(colType == 'string'):\n",
    "        stringCols.append(colName)\n",
    "        \n",
    "print stringCols"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Then, we count the number of invalid values in each column and show the result."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+----+----+------+--------+--------------+--------------+--------+--------------+--------------+-------+--------------+\n",
      "|step|type|amount|nameOrig|oldbalanceOrig|newbalanceOrig|nameDest|oldbalanceDest|newbalanceDest|isFraud|isFlaggedFraud|\n",
      "+----+----+------+--------+--------------+--------------+--------+--------------+--------------+-------+--------------+\n",
      "|   0|   0|     0|       0|             0|             0|       0|             0|             0|      0|             0|\n",
      "+----+----+------+--------+--------------+--------------+--------+--------------+--------------+-------+--------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from pyspark.sql.functions import *\n",
    "from pyspark.sql import *\n",
    "\n",
    "def isInvalidValue(colName):\n",
    "    if colName in stringCols:\n",
    "        return (col(colName).isNull() | (col(colName) == ''))\n",
    "    else:\n",
    "        return (col(colName).isNull() | (col(colName) < 0))\n",
    "\n",
    "transactionsDF.select([count(when(isInvalidValue(c), c)).alias(c) for c in transactionsDF.columns]).show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "As shown, the dataset does not contain any invalid values so we do not need to elminate any records."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Part 3: Exploring the dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### **3.1 General Statistics**\n",
    "\n",
    "Here are some general statistics to give us a better idea of how the data looks like. First, let's take a look at the distribution of transaction type."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Prepare data to show transaction type distribution\n",
    "groupedByTypes = transactionsDF.map(lambda x: (x.type, 1)).reduceByKey(lambda a, b: a + b)\n",
    "\n",
    "# Pie chart, where the slices will be ordered and plotted counter-clockwise:\n",
    "labels = groupedByTypes.map(lambda (x, y): x).collect()\n",
    "sizes = groupedByTypes.map(lambda (x, y): y).collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAeQAAAFtCAYAAADWERnpAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAPYQAAD2EBqD+naQAAIABJREFUeJzs3XecVNX5x/HPc+/dZelFQLAABgQLKioitlgJKPYSNSoo\nGLFHNMnPqIkt0Wg0GrErqBFjjdhAE7BhMEYFGxZQRFQQpEmRnd2de8/vjzOLy7rAlpl7Zu48b1/z\n2nH3zsyXnZ155px7ihhjUEoppZRbnusASimllNKCrJRSSuUFLchKKaVUHtCCrJRSSuUBLchKKaVU\nHtCCrJRSSuUBLchKKaVUHtCCrJRSSuUBLchKKaVUHtCCrJRSSuUBLchKJZiI3CcikYiEIlIpIgtF\n5N8icpqISI3jvsgcV/MSishvMz/vXutnFSLyqYhcWuvxLheRGZnrc+u4z5r3PS7e34ZS+S1wHUAp\nlXPPA6diX++bAkOAvwHHishhxpgIMMBlwL21bruqxnUDHAh8BDQD9gbGisgCY8x9dTxuf8DPXN8L\neALoXeM+y5v2z1IqWbQgK5V8FcaYxZnr3wDvisj/gBexhbq6pbraGPPtBu5HgGU1jnlYREYAuwA/\nKsjGmKVrbyiyLHN1sTFmZaP/JUolmHZZK1WEjDEvA+8BRzf2PkSkP7YYv5GtXEoVMy3IShWvT4Ae\nNf7/OhFZVeOyUkT2qnWb1zM/qwD+BzxmjHkorsBKJZl2WStVvAR7XrjaX4D7ax0zv9b//xxbyEuA\nvsCtIvKdMeZ3uQqpVLHQgqxU8doW+LzG/y8xxny+voMzvq5xzCwR6QlcLSKXG2Mqc5JSqSKhXdZK\nFSEROQDYAfhnA25m1vO9ACjNRi6lipm2kJVKvmYisil2CtKmwMHAxcAzwIM1jmudOa6mNcaY6mlK\nAnTMHBMAOwLnAy8ZY1bXI4ds/BClipcWZKWSbwiwAEgDy7Gjq881xvy91nFXZS413QWcnblugMmZ\n6yF2CtVz2PnL9VFXC1splSHG6GtEKaWUck3PISullFJ5QAuyUkoplQe0ICullFJ5QAuyUkoplQe0\nICullFJ5QAuyUkoplQe0ICullFJ5QAuyUkoplQd0pS6lCpCIeNilMGteACKgCqgyxkSO4imlGkFX\n6lLKIRERYBOgJ3ad6Q61Lu3B2wSCTva4qB2kW1FrXeiAwKRJ114rOvLwwsijAs9bAqwkilYQRcuB\nZcDX2O0Vv65xfYnRNwWlnNCCrFSOZVqzXbFFt1fm0hNKt4HwJxC2WvcWLUJoF9l63NGHjh6054ca\n3Qa7HfEPjWPhZAYzmN3YjXTmvyqqeJRH+ablSjj0UFizBr7/3n5dvjzNt98aVqwIiCKpEbYK31+E\nMV8Shl8CXwAzgQ+AT3SLRaVyR7uslcoiEWkG7ATsBgyA0t3B6wFRs8wRQJcq6O1Bb39tbaYnsBm2\n8JbU7IKup5PpTW8O4IB1vjuRiYYOgXDmmbVvYF/7YQjLl8PixbBkCSxeXMLixVuwZMkWLFq0O/Pn\nRyxbVpL5x4VSUjKHdHoGtkBXX+Zpq1qpptOCrFQjZVq+vYEB9lKyJ8gOYAIIDOwQwoAA+vBDw3gr\noKwkrowpUlC6ga2KfR86drSXOn4K+KxaBXPnwuef+8yd25vPPuvJ3LnHUV5uPzR43hoJgncJw5eB\nV4HXjTHfZ/9fo1SyaUFWqp5ExAd2BYaAvx/4AyBsaX/aswr2LFlbm9lJoJnz11eKlNCsddPupHVr\n2HFHe7F8jLGt6s8/h7lzW/DJJ3vwzjsDWLXqUkRCCYJ3CMOXsAV6mjFmRdNCKJV8zt8wlMpnItIZ\nGAwMgWAopNtCqxAO8GCg2OK7K9AutlZvQ1RQAWVl2b9jEejc2V4GDgQQjAmYNw/ee8/n/ff7M2NG\nP7777rdAJEEwkzB8EZgCvGiMqch+KKUKmxZkpWoQkQDYHVuAD8OeDwZ2SsOhAQwBBvqF8tKpoAKa\nN4/nwUSgRw97OeIIMCZg/nx47z2P997bkRkztmXp0tF43hoReRaYAEwyxqyKJ6BS+a0w3lWUyqFM\nET4I5GTwD4ewNbRNw8E+HIxtIG9acK8Vg6GSyty0kOtDBLbYwl6GDgVjSvjiC3jttRa8+uoxfP75\n8YikxfenEEX/BJ4xxnzrJqxS7hXcm4xS2ZCZ/7srcDIEp0C6A/RKw4mBLcK7BQ0e6JxnqqjCYKBl\nS9dRLBHYait7GTYsYOFC+M9/AqZOHcTMmYMx5m4Jgv8Shk8AjxhjvnEdWak4aUFWRUVEtgJOgpJT\noaondEzDKQGcDOwc1Fpvo6ClSNkrcXVZN1SXLnDssXDssT7Ll8Prr8PUqQOZPn0gUXSD+P5zRNE9\nwAvGmLTruErlmhZklXgi0h44HoLhwEBoHsIxHpwCHBAk9WWwtiDnSwt5Q9q3t93aQ4d6rF4NU6bA\nM88cwty5h+P734rIPcA4Y8znrqMqlSvJfCdSChCRPsCvwDsNaAYHGhgGHOFDARSpJlpbkFu12vCB\n+aZVKzjySDjyyIDZs2HixM78+98Xk0pdKr7/MlF0N/CUMSblOqpS2aQFWSVK5tzwgeBfBAyBTdJw\nXgCjgC7J6Y+uh7UFuXUT5yG71Lu3vZx1ls8rr8Bzz/2UDz/cH99fISL3AjcbY752HVOpbNDtF1Ui\niEhzETkdgo+AybDtQXA/MD+Ay4EubgM6UEFmqm+htZDrUlYGQ4bArbf6PPAAHHtsW1q0uACRuSIy\nVkS2cR1RqabSgqwKmoh0FZGrIZgPcg8c3BteBt4PYDjQzHVEZ8opt1cKuYVcl27d4Mwz4bHHfEaN\nCmjXbhjwkXjekyKym+t4SjWWFmRVkDKF+DaQL6H57+Ds9jAbeMaD/UjSaOnGWttCbtPGbZBcadkS\njj8eHn004Ne/Frp0OQx4U3z/ZRE5KHP6QqmCoQVZFRQR6SAi14E3F9qMgmsCWODD37CbN6hqa88h\nt2vnNkiulZbaEdoPPhhw+eWw1Vb7AJPx/RkicowWZlUotCCrgiAirUTkMvC/hOYXwSXNYJ4PFwMJ\nLziNVFDTnrLB92G//eCee3z+8hfo23cH4Al8/00R2dt1PKU2RguyymsiUiYiv4JgHgRXwnkt4Qsf\nrkYL8YalSOHhgVdkL3MR6N8fbr7Z569/hR49dgZey5xj3tp1PKXWp8heqapQiEggIiMh+BzkJhje\nAeZ4cBPQ2XW8glBBBVLsL/Gdd4a77/a55BLYZJPDEPlYRG4RkTo3gFbKpSJ/tap8JCL7QTATuBeO\n6gIfC9wLdHOcrLCUU44UW+u4Lp4HgwbB+PEBv/ylT1nZ2XjeFyLyfyLiaOcNpX5MX60qb4jIZiLe\nP4CXoX8veBt4TKCP62gFqYIK8DzjOkfeaNYMTjwRHn7Y54gjWuJ51+L7c0TkFzrwS+UDLcjKuUz3\n9GjwP4X2P4f7gGm+3YxJNVaKFEZbyD/Wrh2cfz7cf7+wxx5dgIfwvBdFRIfpK6f01aqcsgs5BDNA\nboSzWsAcH05F/zSbLkUKExT2FpI5teWWcPXVHtddBx077oPIRyJyqYiUuo6mipO+6yknMtOYbgL5\nH2y3HbwlMAYdOZ09FVQQlfjaFbsxAwbAAw8EHH98CSJX4/vvichA17FU8dGCrGInIgdDMBvKzoe/\nCEzX7ukcWMMaKC1xHaMwlJXBqFFwzz1Cz55bA6+LyA0ikqebSask0oKsYpPZAGIMMAn22xQ+8uAi\ndNOx3CinHEq0IDdIz55w++0+Z5whBMFofH+miOzlOpYqDlqQVSxEpK89V1xytu2a/rcHW7mOlWjl\nlNtlJVXD+L4djT12rEfv3t2B10TkOhHRTzcqp7Qgq5wS6zzwZsDWvWCGB+eimz/kXoqUneqjGqdb\nNxgzxmfUKMHzfoPvTxOR7q5jqeTSgqxyRkQ6gzcJuAXOKYHpAfR1HatopEjZc6Oq8XwfTjgBxowR\n2rffGc/7QESOdB1LJZMWZJUTIjIEgo+g3UHwHHALoONj4lRJpbaQs2W77WDcuIA992wJTBCRv4mI\n/nJVVmlBVlklIs3sdCaehwPaw4cBDHUdqyhVUAHN9UNQ1rRuDVdd5XH++eD75+L7/9PFRFQ2aUFW\nWWO7qP1XIDjfbgLxvAddXMcqSmHmP1q0cB0lWUTgqKPg9ts9OnXaHs97T0ROcB1LJYMWZJUVIrIT\nBO9A+/7wmgcXoH9e7qzdC1kLcm707g333huw777NgYdF5DYR0fl7qkn0HVM1mYgcBd4bsN2mMCMA\nXeTItQoq7JWWLd0GSbKWLeH3vxdGjwbPOwvPe15E2rqOpQqXFmTVaJkpTZcBT8LRpfC6D1u6jqXI\nzEEGLci5JgKHHw7XXy80a7Z/5rxyD9exVGHSgqwaRURagPcwcDVcCTzmgb755wttIcds113hzjt9\nOnbsie+/rWthq8bQgqwaTEQ2h2AalBwHjwN/QBf6yC9rzyG3bu02SDHp1g3uvDOgT592iEwVkeNd\nR1KFRQuyahAR2dUO3urUF/7rwbGuI6k6rC3Ibdq4DVJs2rWDm27yOeCAAHgks52jflpV9aIFWdWb\niOwD/lTYsYMdvLWz60hqPbQgO1RaCpdeKpx6KsAfgft1ERFVH1qQVb2IyM/Amwx7NYNXfJ1fnN+0\nIDsmAsOHw2WXQRCcjOc9rVs5qo3Rgqw2yq7d602EwSXwgg96XjLfaUHOEwceCH/+s4fvD8LznrOD\nIZWqmxZktUEichLIP+FoH57ydD3qwlBBBYLo9ov5YNdd4frrPYJgPzzvXyKin2hVnbQgq/USkTOA\nB2GYwMMC+uZeKFKk8PTlnT/69YMbb/QoLd0D35+iC4iouugrVtVJRC4E7oJzBcYJ6KqAhSRFChF9\neeeVvn3tCOxmzXbF918SkfauI6n8oq9YtY7M6luXAzfCxdhtE/XPpNCkSIGnz1ve2WYb+NvffJo3\n3wnff1VEOrqOpPKHvmJVbb8FrrCzNa5FF/woTJmCbFznUHXo1QtuucWnZctt8f2pIrKp60gqP2hB\nVmuJyEjgz/B74FLXcVQTVFBBFPiuY6j12WorGDMmoHXrrTPd1+1cR1LuaUFWQPXUJrkHzsSuTa0K\nWYoUJvC0eyOfdesGN98c0KxZHzzvWREpcx1JuaUFWSEi+4H3GBwN3Ip2Uxe+csoxJToQL+917w7X\nXefjeXsi8pCIaLdGEdOCXOREZGfwJ8J+PjwkoO8HSbCGcp2DXCj69oUrrvCAo4BbdO3r4qUFuYiJ\nSC8IJsNOzeyiH7rcblKkKDeUlLiOoeprr73goosEOBu4xHUc5YYW5CIlIl0heAl6tIV/6XKYCVNO\nOTTTD1gFZehQOO00gD9mBliqIqMFuQjZVYKCKdCpK7wYgE6FTJoKKkQLcgE65RQ4/HCAe0TkMNdx\nVLy0IBcZEfHAfwjK+sCUALq5jqRyIEUKynTQbsERgfPPh733BpEnRGR315FUfLQgF58/QDQUHvVh\nO9dZVI5UUgnNdSOQguT78PvfC9tu6+P7z+jCIcVDC3IREZHDgcvhauAQ13FUjhgMVVRpQS5kpaVw\n1VU+rVp1wPOeEBEdoVcEtCAXCRHZBvyH4cgIfuc6jsqhSioxGC3IhW6TTeDqqwNgL+B613FU7mlB\nLgIi0gaCZ6FnKfzd06c92VKk7JWWLd0GUU23ww5wzjkCXCAiJ7qOo3JL35kTzg7i8h6EZlvBs4FO\nb0o+LcgJc9RRcNBBBs8bJyI7uI6jckcLcvJdCtHh8IgPvV1nUTFYW5BbtXIbRGWHiF00pHv3Enz/\nWd2IIrm0ICeYiBwKXGk3izjUdRwVEy3ICVRWBn/8o0+zZlvgef+wPV8qafRJTSgR2czONz7UwGWu\n46gYrS3IrfX0RKJsthn84Q8+UXQwuj9qImlBTiC7OL1/P3RoAffrIK4iU0GFvdKmjdsgKvt2392u\n5iVyhYjs5jqOyi59p06mURAOggcC2MR1FhWzcsrtFS3IyTRsGPTsafD9f4iIzm1LEC3ICSMiW4N3\nE4wCDnYdRzmgLeSECwK49FIfkZ8A17iOo7JHC3KCiEgAwUPQLYAbXMdRjqw9h9yihdsgKnd69IAz\nzvCw85P3dx1HZYcW5GT5Pwj7w0MB6AjbYpUihYcHnr68E+2YY2CHHUJ8/0G7+I8qdPqKTQgR2QXk\nSrhYYE/XcZRDKVLorJgi4HlwySU+QdAFuMl1HNV0+qpNABEpg+Bh6GvgCtdxlGMVVGhBLhZdusB5\n5/nACN0/ufDpqzYZ/gj0gn8EUOo6i3KsnHLwPOM6h4rJIYfAwIERvn+fiHRyHUc1nhbkAiciO4Jc\nCFd70Nd1HJUHKqjA+PrSLhoi8JvfeJSVtQP+4jqOajx91RawzAIgt0HPEC50HUfliRQpTOC7jqHi\n1KEDnHGGDwwXkT2yffcicp+IRCISikiFiHwqIr+vvYSniHwiIuUi0rnG98pE5GMRua2O+/2riMwW\nkeYiMjLzGO/XcdyJmZ/NrvG9kTUyRTUuK2scMz7zvQtr3d+xIlKVuf5grdvXvswmJlkvyCKyqYiM\nEZE5IpISkXki8oyIHFDruN+JSFpELqrjPjwRuTjzJK4RkaUi8oaIjKhxzH0i8mQdt90380us16jD\nzGONFpH3M39Iy0RkkojsWeu4y0XknTpu3z3zeDvW+qOt64n9vD6ZGuAECPeG27SrWq2VIkVUEojr\nHCpmQ4dCz54hvn+7iOTiE9nzQBegF7Ylfjnw6+ofisheQDPgCeDU6u8bY1LAMOD0mnUgc/w5wHBj\nTGY1G1YBm4vIrrUeewQwr45MSzOZal5+UuPnBigHficirWt9v/q0ztk1brt55nsn1fjewLp+GbmQ\n1YIsIt2BGcB+wEXYPtQhwMvArbUOPw24DvuLru0K4FfY9Vq3zdzfXUB9dzlpyPmzR7GLPd8EbAPs\nC3wFvCIih9fzfqu/fz72Ceya+QownB+e2KwtdWf/uIKb4SgDP8vW3aoEKKccSgLXMVTcfB9Gj/YJ\nw37AyBw8QoUxZrEx5itjzN3AFOCIGj8fCfwDGE+t93VjzFvY9/txItLKDkRlHPBXY8x/axxaCTxS\nM7+IdAP2Ah6uI5PJZPq2xmVJrWP+hS3cF9f1jzLGrKq+LfBt5tsratzfsg38TrIq26/aO4AQ2C3z\nqajaxyIytvp/RGRfoAz4A7aLZaAx5o0axx8G3G6MqdkC/iDLWRGR44FjgEONMZNq/GiUiGwC3Csi\n3Wt8elvvXYF9YrGf8KrvHzJPbHaTA3AZeB3hJm0JqXWsYY2htFT/LorR9tvD4MGGKVOuE5EnclxM\nUmTW5s20Po/DNjpmA21FZC9jzLQax18FDAX+BqwEKrA1oCaDLdSTReQCY0wltvH2HLaoNkYa27h7\nQETGGGMWNvJ+ci5rLWQRaQ8MBm6tVYwBMMasrPG/I4CHjTEh9lPP6bUOXwgcICIdGxOlAceeCMyq\nVYyr3Qh0BAY1IkNOiUgfO5DrMg+6u46j8kyKFJTqKYyidcYZQklJa+DqXD2EiByEfb9/MfOtE4DZ\nxphPjDER9n19nVa6MSaN7br+BXAmcIoxpqr2fRtjpgNfA0dnvnUqtkjXpaOIrBSRVTUuT9dxn/8E\nPsR2s+etbHZZ98IWw1kbOijzSepY4MHMt8YDx4lIzXX+LgQ6AQtF5D0RuUNEhtRxd4fVeiJWAXUV\n1/XpDXy8np99XOOYvCEiAv6tsCXwG9dxVB5KkRKaNXMdQ7nSoQOMGOEDZ4nITlm85+r32xQwEVt0\nr8z87DTse3m1fwA/F5GWNe/AGPMh8BTwvDHmvQ081jjs3OoDAR/b7VyXZcBOtS6j1nPsxZn77LWB\nx3UqmwW5vi3TXwCfGWNmAmSelC+B46sPMMZ8bIzpC+wOjMUW52dF5O5a9/USsCPrPhm1W9vZyp0v\njoDwIBgT2F5/pdZVQYXd0F4Vr6OOgi23DPG8OyRz7iwLqt9vewHNjTEjjDHlIrItduDT9SJSlRm9\n/F+gObblXFs6c9mQ8cA+wO+Bvxtj1jd+JzLGzDXGfF7jUmeXtDHm5cy/4dqNPLYz2SzIn2L7/7fZ\nyHEjgO2rn7jMk7ctdQzuMsZMN8bcYow5FtttMTIzcKza97WfDGB+AzLPzjx2XbbLfK1u8a8E2tZx\nXDvsv3tFAx63UUSkOQRjYHBkT7Mr9WMpLcgqCOBXvwqIoj2wjaBsqH6//TrTLV1tJPAqP24c3UQj\nB5dlBmZNxBbl+5qUel2/A44CBmTxPrMmawXZGLMc261wjtSxR6eItBWRvsCu2JHMNZ+4/YE9RGRD\n3cPVXcgtN3BMQz0CbC0iQ+v42UXAEuxIQrCFeQv58Uo4u2IHN3yZxVzrczZEm8MtXuE17FVcqqjU\ngqxg111h770Nvn+diORkUIGIBMApwD8yPZsfVV+Ae4GBmRZ0Y5wEbGKMmbPhCLJp7cv6DjbGvIud\nWXNeIzPlVLbnIZ+D7e9/U0SOFpFeIrKNiJyP7cIYCbxpjJlW84kzxrwGvJ35OSLyuIhcICIDRKSb\niOyHnTY1C/ikHjnqVa2MMY9gz2c8ICIjMnOKdxSRu4BDgZE1Rlj/K/P4D4vIHiKylYgcix04cfMG\nulSyIjPN6VIYKXl2WlvlkTRpQkLdelFZI0cKYbg5DT+VV1+HAx2w76PrMMZ8AnxE41vJFcaY7zZy\nWAdgQY3LN8ACEemwgdtchq1TG5vGGrusFmRjzFxgF+y84xuwU5X+jZ0o+1vsJ54n1nPzfwLDMhPa\n/4UtiM9gi+B92Cd2cK2ukvVGaUDs47CbfF+ALfavYkdM7WuMebbGvy3M/Du+xA5Y+AA7Yu8mfjx0\nvzE5NuZXQFt7SkWpuuleyGodPXrAoEEG37+i1sDZBjHGnGaMObqO7z9pjCkxxixez+36GmN+Xet7\npxhjfl7HsWONMZ1rf7/Gz280xvSudbxf6+Jlvi5b32Nlut2bGWN+NPLRGBNmbt+QwcFZIzlu2Kks\nsFPK/C/hnFZ2Cp9S6xI8zuM89mEfjuM4GD4cTj3VdSyVD+bPh2HDDFH0W2PMDa7jqPXTtawLw0VQ\n0gIucZ1D5bkKKuyVVq3cBlH5Y/PN4ZBDBM+7VET0DyOPJbogZ9akXlXHZaWI1LmMWr7JtI5Hw3ke\nrHesglJAZtlMgJbZHPuoCt7JJ4NIW+As11HU+iV9wduR2LlwdYltfdImOh+CMjvoW6kNW9tCbt16\nwweq4rLppjBkiPDCCxeLyG3GmDWuI6kfS3QL2RjzTa0J4zUvGxu955yItAH/IjhTW8eqXtYO6mpT\nr83OVDE56SSIovasfyUr5ViiC3ICnAPSUpfIVPW1tiBrC1nV1rUrDB4Mvn9JXWtFKPe0IOcpuz1Z\n8Bv4pffDFp1Kbdjagty2rkXlVNE76SQhDDuSvdW7VBZpQc5fx0O6PYx2nUMVEO2yVhu0xRaw++4R\nvn9BFte4VlmiBTlvBefDoAi2dh1EFZAUKQSxaxkrVZejj/YIw77AXq6jqHVpQc5DIjIA0rvYqU5K\n1V8FFXj4rmOofNa/P3TtmkYkL9dzLmb6hp+X5BzYIg2HuA6iCkyKFNoTqTbI8+CYYwLgGBHZzHUc\n9QMtyHlGRDqCnAjnBWhLRzVQipR9w1VqQ4YMgdJS0ClQeUVfuflnJPh+HdtDK7VRKVIYz9MF6tWG\ntWwJQ4b4+P45udqaUTWcFuQ8Yne6Cs6Dkzzo6DqOKkApUpjA1z5rtXFHHQVhuAl2xzuVB7Qg55eh\nkN7cbiutVMNVUEEU6KkOVQ/du0O/fiG+/yvXUZSlBTmv+OfBriH0dx1EFahyyqFUpzypejrmGJ8w\n3E1E+rmOorQg5w0R6QbhQXCeNm9Uo9mCrKcEVT0NHAitWqXRlbvyghbk/PFzKI3gaNc5VAFbwxqj\nBVnVWxDAAQcE+P7JIqL1wDF9AvJGyUkwVEA3BVCNlyIlWpBVgxx4IIRhV2BP11GKnRbkPCAiPaGq\nH5ygo2NVk6RIQVmZ6xiqkPTtCx06pIETXUcpdlqQ88PxUBbBUNc5VIGrpBKaNXMdQxUSz4NBgwJ8\n/0QR0RGBDmlBzgslJ8HhAi1dB1EFroIKbSGrhtt/fwjD9sABrqMUMy3IjonINlC1nXZXq6aKiKii\nClq0cB1FFZreve2GE9pt7ZQWZPeOhxYhHOw6hypwlVTaK1qQVUOJ2G5rzztORLSLxREtyA7ZDcJL\nToKjfdDXgGqaFCl7RQuyaowDD4Qoaom2DpzRguxWX6jaGo53nUMlwNqC3FLHIqhG6NYNundPA0e4\njlKstCC7dRS0DOFnrnOoBFhbkFu1chtEFa6BAwN8/xDRTbWd0ILsVDAYDvJAF3JQTacFWTXZbrtB\nGHYC+rqOUoy0IDsiIq0gHACD9JOoyooKKuyVNm3cBlGFa4cdoKQkAga7jlKMtCC7sw+YAA5ynUMl\nxNoWcmtdflU1Umkp9OsHnjfEdZRipAXZnYNg0zT0dp1DJcTagty2rdsgqrANGOBhzE9FpLnrKMVG\nC7IzJUNgSADaY62yY21B1i5r1RT9+4MxJcBPXUcpNlqQHRCRznZ1Lu2uVtmTIoUgOg9ZNU337tC+\nfRqd/hE7LchuZNaLPdBtCpUotiDrS1o1kYid/hQEuttNzPTV68ZB0LsKurrOoRKkggp0j3mVFf37\nQzrdR0Q2dx2lmOirN2aZ5TKHwJAS11lUsqRIoes5qKzo16/62h4uYxQbLcjx6w5Vm2t3tcq2FCnw\nfOM6h0qADh2gQ4cqoL/rKMVEC3L8drVfBrhNoRLJ+PqSVlmy7bYBnre76xjFRF+98dsFOqahi+sc\nKoGiEl/7rFV2bLONAP1FBybERn/RsfN2gd30965ywpT4riOopOjTB6KoFdDTdZRioYUhdn5/2EV/\n7yo3SnSsoMqSPn2qr+3mMkYx0cIQIxHpAlUdYWfXUVRSlerOYSpL2rSBzp11YFeMtCDHa+d1viiV\nbVqQVTZtv30Jvj/QdYxioQU5XjtDqxC2cp1DJVWzZq4TqCTp0weM2VlEdHBCDLQgx2tn2zrWgbAq\nR8rKXCdDbewnAAAgAElEQVRQSWIHdpUB27iOUgy0IMeqdAD010+aKne0IKts2mptb54W5BhoQY6J\niLSBym56/ljllBZklU1t2kBZWYhOfYqFFuT49LJf9IOmyiHdelFlkwhstlmEFuRYaEGOT3f7pYfT\nECrhtCCrbNtyyxI8b2vXMYqBFuT4dIdmEXR0nUMlWatWrhOopNlsM/C83q5jFAMtyPHpDluEOsJa\n5ZS2kFW2bbYZpNNdRUQnueeYFuT49ICegesQKuHatHGdQCXNZpuBrRXdHSdJPC3IsSntBT20eaxy\nq2VL1wlU0tiCDDqwK+e0IMcm6qYfMFXOaQtZZVunTuD7Bi3IOacFOQYi0grSbbQgq5zTgqyyzfeh\nc+c0WpBzTgtyPHTKk4pH27auE6gk6tTJA7q4jpF0WpDj0X2dL0rlik57UrnQtq2PyCauYySdFuR4\nbG6nO3V1nUMlmIcHgQ7kVznQpg34fifXMZJOC3I82kKLEHRfCZU7oi9nlSutW4NIB9cxkk5fwfFo\nDa0i1yFUsonoy1nlSJs2EEXtXcdIOn0Fx6M1tHadQSWdp9PcVY60bg1h2Fr0U19O6S83Hm2gjb5b\nqpwyvp4SUTnSujXYgTA6jD+HtCDHozW009+1yikT6J+YypHWa3v49DxyDukrOBbSFtrq71rlVFSi\nI6xVjvyw4IyeR84hLRKxCNrqOWSVc1qQVa5oCzkWWpBjIW1BlzRUOVZS4jqBSqqysuprur9nDmlB\njkcbbSGrXDIApbpdrco5HZyaQ1qQYxG10oKscstAs2auQ6ikkrV1WAtyDmlBjocPen5P5ZgWZJUr\nWpBjoQU5Hmmocp1BJd0P5/mUyi7ROhwHLcixkDSkXYdQiXWO/fLtt7B4sdsoKum0MueQFuRYaEFW\nubAE6A3cbl/JX30Fw4fD5MlgjONsKqG0IOeQFuR4VGmXtcqup0E2A/9TOAL4BRBFlJSXwzXXwGWX\nwbJlrkOqpNBzyLHQghwPbSGrLImA00COhA5VMArYGehlZ7t3BkYD8sYbtrX8yisuw6qk0IIcCy3I\n8dCCrLJgAfAT4H7ohy3GnX/4qRkM84EdgQ+iiK6rV8OVV9rLihUO8qrE0EFdsdCCHA/tslZN9ChI\ndwjmwdHYbura64BsB9ISrgK2Ab4GzgaYOhWGDYNp0+KNrJIjDNdecxkj6bQgx0NbyKqRIuAEkBOg\nUxrOwjaB18MMgrnA49gX923A9Cii46pV9rzytdfC6tVxBFdJsmZN9bVVLmMknRbkeFRqQVYNNw/Y\nEngU+gO/BDbZyE36gVcGV2JLOcAuwCJjOA1gyhR7bvnNN3MVWiXR999XX1vpMkbSaUGORVQBFa5D\nqILyAHg9oXQB/BwYCtRz74joAPgEeLbG9zxgHPB6FNFu+XL4v/+DG2+s2fJRav3Ky6uvaQs5h7Qg\nxyK9FJbqxFBVDxFwJHAqbBraLurtGngXA8Avta3k2n90ewBLjeEEgEmT4NRT4d13m5hZJZ62kGOh\nBTkeS2CxDoZQG/Ep0AV42lbOkTR6O/hwH3gHmFzHzzzgYeClKKL10qUwejSMGQOpVOMeTCWfFuRY\naEGOx1JYrC1ktQF3gLcNNFsMJwKDadp+JHuBH9hW8vrsDyyLIo4AmDABRoyAmTOb8KAqsVauBNt9\no/PnckgLcjyWwjKdyKfqkAaGAGfDZpGdp9QnC3frQbgHvA5M3cBhAfAUMMkYWixaBOefD3feCZWV\nWQihEmPlSvD9Fcbomqy5pAU5HkthdaAjrdW6PgTpDPwL9gFOA9pm8e73A9+385I35mBgaRQx2Bh4\n7DEYORJmzcpiGFXQVqwAEV2LNce0IMdjkR1eozvxqGp/BdkBypbDKcCBgJ/lh/Ah7A8vAm/V4/Ay\n4AXgSWMoW7AAzjoLxo2DKl3UpuitWAHG6BtYjmlBjsc363xRRawS2A+4CLoZ20XdM4cPNwgCD65u\nwE2OwraW9zUGxo+HUaNgzpxcJVSFYPlyQxgudB0j6bQgx0MLsgJmgHQy8KodUTUcaJ3jhwwgvZOd\nk/xBA27WAngFGG8MpfPm2aI8fnzNJRRVMVmwIA184TpG0mlBjsciwGhBLmZ/AukPLVYKpwL7Et+r\n72AIBP7UiJueBCyOIgaGIYwda7ux583LdkKVz6IIFi0KgM9dR0k6LcgxMMakoWS5FuRitAY7qfgy\n+Emmi7pHzBFKIb09PAbMbsTN2wD/Be4Bgjlz4PTT7cAvbS0Xh6VLIQwFu0y6yiEtyPGZp3/Pxea/\ndhS1vAEHYZubLR1FGWrHjF3bhLs4Hfgmitg5nYY77rBTpObPz1JAlbe+WduQ0BZyjmlBjk3VTPhQ\n5z0VjUtB9oRW38MIYG/cvtqaQ7oPPEjTTgR2BGYAtwD+rFl2MZEJE2y3pkqmBQuqr33hMEVR0IIc\nn0/skv86rz7ZVmP3V7oGemPXot7SbaK1DrNfrs/CXZ0HfB2GbFdZCbfcAhdeCAt1EG4iLVwIQbDU\nGKM7keSYFuT4fAIrA52LnGQv2y5q7x27+NYJ2OHK+aIVhD+Be4EFGz1447oAHwJ/BryZM+1GFRMn\ngi7mlCwLFoAxOu8tBlqQ4/OJ/fKx2xQqR0aDHABtyu2mEAOBfFws9XC7IPGNWbzL/wPmhiG9Kirg\nhhvs1o6L9YNnYixYEBKGn7qOUQy0IMfnM5BobV1WCbEc2B64GbbFdlFv7jbRBrWDsBvcDizJ4t12\nw+5V9XtApk+H4cNh8mRtLSfB/PkROqArFlqQY2KMqYSSeVqQk+R5kK7gfQSHAsdh15/Md4dBFXBz\nDu76KuDTKKJbeTlccw1cdhks0yWQC1Z5OXz3XQlakGOhBTlWVR/Ax9pkSIQzQQ6BdhVwBtCf/Oyi\nrksnCLvagvxdDu6+JzAP+DUgb7xhW8uvvJKDR1I599ln1dfedRmjWGhBjpX5BD7QqU8F7VugF3AX\n7ACciR3dVGgOh3Lgthw+xF+AD6KIrqtXw5VX2ssK3U63oMyeDSJV2PF7Kse0IMfrE1hQYldvUoVn\nAsgWEMyBI4GjgWauMzVSV4g626K5OocPsz3wNXaBMqZOhWHDYNq0HD6iyqrZs8HzZhpjdMuvGGhB\njldmiPVHblOoBoqA4SBHwyZVMAro5zpTFhwKK4G7c/wwHrYlPj2K6LhqlT2vfO21sDqXHwVUVnz8\ncRVh+D/XMYqFFuR4vQeShjdc51D1Nh/YCvg77Iwtxp3cJsqabmA62OU0UzE83C7AImM4DWDKFHtu\n+c03Y3hk1Sjl5fD11yXAdNdRioUW5BgZY8rBfwe0y64wPAzSA0q+hGOAw4ESp4Gy7xBYCtwX08N5\nwDjg9Sii3fLlds7yjTfCGj2Nk3fmzKmetqYFOSZakGOXngpT9XxMXouAn4P8Ajqn7cCtHVxnypFe\nQFv4I3YqVFz2AJYawwkAkybZVb7e1YG8eUUHdMVOC3L8XrcDu752nUPVaR6wBfA47Ab8EtjEbaJc\nM4PtUprjY35cD3gYeDmKaL10KYweDWPGQCqODnS1UbNng+9/aNdQUHHQghy/19f5ovLIWPB6Quk3\ncDxwCBC4zhSD7UBawtWAix2O9wOWRRFHgt05asQImDnTQRK1jo8/riKd1gFdMdKCHDNjzEIo+UrP\nI+eTNHYrpNOhS2iXv9zWcaSYmUF2t+7HHT1+AEwAJhlDi0WL7F7Ld94Jldo4c+L77+Grr0qAt11H\nKSZakJ2oegVe0wVC8sIsoCvwHOyJ3RiivdtETvQDrwyuxJ5Bd+VgYGkUMdgYeOwxGDkSZs1ymKhI\nvfNO9YCul1xHKSZakN14Hd7z4XvXOYrcrSDbQdkS+AXwM8B3ncmd6AC70vqzjnOUAS8ATxpD2YIF\ncNZZMG4cVOlYyNjMmAFB8JUxRtewjpEWZDemQSTwluscRSoNDALOgy0i20Xd23GkfDAA/FK4AsiH\nBdePwraW9zUGxo+HUaPsVByVe//7X5p0epLrGMVGC7IbH4H/PfzHdY4iNBOkMzAFfgqcCrR1myif\nhPvYXQQmuw6S0QJ4BRhvDKXz5tmiPH48hC6GnxWJb7+FBQsC8ufPoGhoQXbAGBNCNAWe03eVWP0F\nZEcoWw6nAAdQ1F3UddoL/MCeS84nJwGLo4iBYQhjx9pu7HnzXMdKprffBttJ8rLjJEVHC7Iz5ml4\n07e7B6ncSmGbw7+F7sbudNDTcaR85UG4h52UN9V1llraAP8F7gGCOXPg9NPtwC9tLWfX228bfP9d\nY4xuZB0zLcjuTLTDGPU0TW69nemifs22iIcBrR1Hynf7ge/DVa5zrMfpwDdRxM7pNNxxh50iNX++\n61jJEEXw1lshYfi86yjFSAuyI8aYbyF4C57Oh/EzCXUlyABosQpOwzaS9S9+43wI+8OL5O+ww47A\nDOAWwJ81yy4mMmGCLSiq8ebMgdWr9fyxI/r25FT6KfhXFM9eO8VkDTAAuAJ6Zrqou7tNVHAGQeDZ\n1bvy2XnA12HIdpWVcMstcOGFsHCh61iF6+23wfNS2LMDKmZakN16Bsp9HTuRTdNAOoG8ZecV/wJo\n6TpTAQogvZOdk/yB6ywb0QW7+8GfAW/mTLtRxcSJ1QtbqIaYNi0EXjbGVLiOUoy0ILv1kV1G0/VS\nDEnxO5C9odUau+LWnuhfeFMcDIHYnaAKwf8Bc8OQXhUVcMMNdmvHxYtdxyocixfDhx/6RNEjrqMU\nK327csgYY6DqSZiQzo+lGArVSqAf8Ge7wMfZ2A2bVNOUQnp7u751oSxe2Q34FPg9INOnw/DhMHmy\ntpbr4+WXq7dbfNp1lGKlBdm9Z2FhYJdjUA33Isim4L1nF0I+AWjuOlOCDLVTta91naOBrgI+jSK6\nlZfDNdfAZZfBMp3Fs0EvvpgGJhljVriOUqy0ILs3FfzV+qG0MX4FchC0Tdm5MLsD4jpTwjSHdB+7\nV/IXrrM0UE/s7ta/BuSNN2xr+ZVX3IbKV998A7NnBxjzsOsoxUwLsmPGmCoIn4QHtNu63pZh90e8\nBbYDzgQ2c5so0Q6zX653m6LR/gJ8EEV0Xb0arrzSXlZoI3AdL79cPbr6OddRipkW5PxwH3wR6NrW\n9TERpCv4n9hCcSx2eyCVO60g/AncCyxwnaWRtge+xg4vYOpUGDYMpume5GtNmZLGmGeMMboFnUNa\nkPPDVAi+gnGuc+S5M0AOhfaVcAawK9pFHZfD7T7JN7rO0QQecBswPYrouGqVPa987bWwerXraG59\n+SXMnRtgjI6udkwLch4wxkSQvhceDaHI3xzqtBB7RvAe2AnbRb2p20RFpx2E3eB2YInrLE20C7DI\nGE4DmDLFnlt+803HqRyy3dVrAF0u0zEtyPnjASj37CQT9YMnQLpB8LndIPdIoNR1piJ1GFQBN7vO\nkQUetj/q9Sii3fLlds7yjTfCmjWuo8XLGJg8OU0U/dMYo0sGOqYFOU8YY+aB/wqM1a1rANtBejLI\ncdCxCkZhW8fKnU4QdrUF+TvXWbJkD2CpMZwAMGmSXeXr3SKagvjeezB/fgA84DqK0oKcZ8KxMM2H\nz1wHcexroAfwkO1fPAPo5DSQqnY4lGPPxSaFBzwMvBRFtF66FEaPhjFjIFUEDcannjL4/mfAS66j\nKC3I+WYC+N/D/a5zODQepIddUfRY7EjqEqeBVE1dIepspxIlbbTD/sCyKOIIsDtHjRgBM2c6TpVD\nS5bAa69BGN5iVw1UrmlBziPGmDUQPgRj01BsPdcRtgKfApuGcBbQ13EkVbdD7WKld7vOkQMB8BQw\nyRhaLFpk91q+806orHQdLfueew6gAvi74yQqQwty/rnPLqX5b9c5YjQX2Bz4JwzErrrVwW0itQHd\nwHSwy2kmtVP3YGBpFDHYGHjsMRg5EmYVyore9ZBOw9NPp4mi+3WpzPyhBTn//A+Cd+GGItlp/V7w\nekGzhXYd6iHYZorKb0NhKcmeOV8GvAA8aQxlCxbAWWfBuHFQVeU6WtO99hp8912Ancmm8oQW5Dxj\nz+Wkr4WXPJjhOk4OpYGhwC+ha2S7qLdxHEnVX0+gLfwJOxUqyY7Ctpb3NQbGj4dRo2DOHNexmmbC\nhBDff90Yk+/bXRcVLcj56Um7ctf1CR1o8TFIF2AS7AWMANo5jqQazAy2S2mOdx0kBi2AV4DxxlA6\nb54tyuPHQ1iAYz3mzoUPPvAJw1tcR1Hr0oKch4wxaUhfB49hz68myRiQ7aHZUjgJGITd308Vnu1A\nWsLVFM8QxJOAxVHEwDCEsWNtN/a8ea5jNcxTT4HvLwEmuI6i1qUFOX/dB/4K+KvrHFlSCRwEnA9b\nGrvK/9aOI6kmM4PsR8ZiWl+uDfBf4B4gmDMHTj/dDvwqhNbysmXw/PMRYTjGGJPAoeOFTQtynrJT\noNI3wz1R4a8e/D7IpsCLsC8wHPuupgpfP/DK4ErsxLVicjqwMIrYOZ2GO+6wU6Tmz3cda8MeeQTC\nsBwY4zqK+jEtyPntNjuks5DXRboepB80/w6GYVdf0C7qRIkOgE+AZ10HcWAT7NDLWwB/1iy7mMiE\nCRDl4ceTZcvgqaciouivxpjlruOoHxNdoCW/icgYaH8mfB3YoSWFIoXtop4GWwHHAK3cJlK5418D\nO1Ta4lSsO2IuBA4EPgLYaSe4+GLo0sVtqJruuAOeeOJ7oqibMWaZ6zjqx7SFnP/+Ct/5hbWc5lsg\nnUCm2XeoU9BinHDhPvAuMNl1EIe6AB8Cfwa8mTPtRhUTJ9odlVxbvhwmTIiIopu0GOcvbSEXABHv\nYehyLMwJoLnrOBtxBciV0BI4DujuOI6KR2RbybunYZrrLHngS+xn0c8AdtsNfvMb6ORwh5Q774TH\nH19DFG2pBTl/aQu5IJg/wEKxZ6ry1WpgN+BK6IUdRa3FuHh4EO4BrwNTXWfJA92AT4E/ADJ9Ogwf\nDpMnu2kta+u4YGgLuUDYc8ktz4IvfOjoOk4t/wEZDKyx84r3oHhPJBaz0LaS9wthiusseeQzbGv5\nS4A994SLLoIOMS7Wftdd8Nhj5ZnW8dL4Hlg1lLaQC8fVkKqwixXmk9+C7AOt18BIYE+0GBcrH8L+\n8CLwlusseaQXMA/4NSBvvGFby6+8Es+Df/cdPPlkdetYi3Ge0xZyARGRSyG4GmaLHbrs0nfAT4EP\n7BrUR5D/p7dV7qUhuAYOjuAZ11ny0IfYTqRvAPbbDy64ANq2zd0D/u1v8MwzazIjq7Ug5zltIReW\nm4DFcInjT1GTQbqC9wEcAhyPFmNlBZDuZ+ckv+86Sx7aHvgaOAdg6lQYNgym5WgY3Jw58PTThij6\ngxbjwqAt5AIjIiOBe22nYH8HCc4DuRXaYgtxVwcRVH6rhOBaOMrY1dhV3WYAg0VYYgz87Gdw3nnQ\nKkvzA42BCy4I+fDDLwjD7XSZzMKgLeTC8wAEn8BFIcT5YWoJ0Ae41X7MPwstxqpupZDeHp4AZrnO\nksd2ARYZw2kAU6bYc8tvvpmdO586Fd5/3ycMz9ViXDi0hVyARGQo8BxMxPYZ59ozIMeBV2m3MN4Z\nHbilNqwcguvs7kj3u85SAP4LHCLCd8bAoYfaXaRaNHJlvlQKTjklzbJl/zZhODSrQVVOaQu5ME0C\nfypckIaKHD5MBJwOcgR0qIRR2I/1WozVxjSHdB+7V/IXrrMUgD2ApcZwAsCkSXaVr3ffbdydPfoo\nLF0KUfSr7CVUcdCCXICMMQbCc+AzgWtz9CgLgZ7AWNgJW4w75+ihVDIdZr9c7zZFwfCAh4GXoojW\nS5fC6NEwZoxt8dbXwoUwfnyEMTcaYz7LVVaVG1qQC5QxZiaYa+BPxk6myKbHQLaE4As4CjgSKM3y\nQ6jkawXhT+y+wQtcZykg+wPLoogjwO4cNWIEzJxZvxvfeafBjqjOtwULVD1oQS5sfwLmwGkhZGNz\n9Aj4Bcjx0CkNZ2Jbx0o11uF26OGNrnMUmAB4CphkDC0WLbJ7Ld95J1RuYHzWu+/Cq68KYXiRMWZV\nXFlV9uigrgInInsDr9n9xs9twj19iV1ma76dTTUYKMlCQKXGQdmX8BX5t+hrIUhhO6n+JQKbbw6X\nXQZ9+tQ6KAWnnZZm8eK3CcO9jDF5uCGz2hhtIRc4Y8x/gDvgt2FmtdxGeAC8n0DJfLtD06FoMVbZ\nczhUATe7zlGgyoAXgH8aQ9mCBXYE9rhxUFX1w0HjxsGiRRFheKoW48KlLeQEEJG2EMyCQZ1golf/\nYdARcAzwlN3M9edAjGveqyJyN7RcYFepauc6SwFbg53o+KoI9OgBl15qW8fnngvwG2PMDU4DqibR\ngpwQInIE8JQdp3lCPW4xB9tF/S0MBA7CnrhSKhe+Ae8uuBK4zHWWBHgIGOF5VIpA69aGVaumE4YD\njTHZGEyiHNGCnCAi/hPQ7giYHcAmGzjyLpCzoTSyo6i3iSuhKmq3Q9tvbSs5SwtEFrWV2LXzFvp+\nSBjuaIz5yHUm1TR6DjlRonNhRTmcZepeVjMNHAycCZtFdvlLLcYqLofaInKX6xwJMR27WgBh+Fst\nxsmgLeSEEZHjgUdgLDCixk8+BPkpmGWwN3ayo+8koipmt0DHZXbEdZnrLAVsBbAdpBfB6yHsrwO5\nkkFbyAljjHkUZCycE/2wtP/NIDtAs2VwMvZ8sRZj5cJQWAqMc52jwJ0LZhFUhDBMi3FyaAs5gUSk\nJQTvwTbdYZMAXoXu2AHVbVynU8VOboKuK+wa1zq7ruEeBIbZq6caYx5wGkZllRbkhBKRftjTTB77\nAT9F+0NUfvgIeMy2kk9znaXAvA8MgKgS/m5ghNE38ETRt+iEMsa8S/V6tp3QZ1rlj+1AWsHVZGfB\n12LxHXAEpEP4yMDZWoyTR9+mk+1yhMd4ipAlrqMo9QMzCOYCj7sOUiAiYBhEX0EqDUcaY8pdZ1LZ\npwU5wYwxBsPphHzBI6TZwLr0SsVqJ/Ca24VCdETSxl0PPAteCCcaY+a4zqNyQwtywhljVhFxJEsJ\nmYipc3qyUg5E+8MnwLOug+S5F4FL7MICfzLGPOc6j8odLchFwBgzE8MveQ9huus0SmUMAK8UrqDu\nZWyUXdXs55D24CXgctd5VG5pQS4SxpgHgTuYiOEz12mUsqJ94F3g366D5KFK4GgIV8LiEE7QdaqT\nTwtycTkfeIFHiFjgOopSwF7gB3CV6xx5xgAjwEwHkxnEpcMyi4AW5CJijEljOI6IdxlPmuWuE6mi\n50G4B7wOTHWdJY9cAjwEEsEpxpg3XedR8dCCXGSMMd8TcTAp5vN30nzvOpEqevuD72srudrtwJ/t\n1YuMMY84DaNipQW5CBljviXiIFawkn8Q6nQo5ZQHYX87mrjYm4ITgHPt1ZuNMX91GkbFTgtykTLG\nfEbEwSygin8S6WRQ5dQgCDy7elexeh04ASKx66Vc5DqPip8W5CJmjHkTw3HMQngenXui3Akg3Q+e\nw67XXGw+AQ6BMITXI93BqWhpQS5ymYUGRvEW8B/XaVRRGwKBwB9d54jZN8AgSH8Pn4ZwuDEm5TqT\nckMLssIYcw9wFS8C77lOo4pWKaS3hyf4YSfvpFsGDIZwISxNw8+MMTr3oYhpQVbVrgDG8RSGT11H\nUUVrKPjAta5zxGApsD+EH8GqNAwyxnzlOpNySwuyAjIbUcCZwEQexvCR60SqKDWHdB94EPjCdZYc\nWgzsC+mPYEUIPzXGfOA6k3JPC7JayxhTheFoDI/zOEa7r5UTh4FgdzhKom+xxXgWrEhrMVY1aEFW\n68gU5V9gGMcEdGKoil8rCHvCPZC4FV4XAT+F9KfwXRr2McZ86DqTyh9akNWPZBax/yVwM5PQ0dcq\nfofZWXg3us6RRd8A+0B6DixPw97GmI9dZ1L5RQuyqlPmnPKFwNVMwS6jpPOUVVzaQdjNLiO52HWW\nLJiPLcZz7WjqvYwxxTKQXDWAFmS1Xsb6A/BbXgNeAF3RS8XmCKgCbnado4m+wBbjebA4U4x1HoOq\nkxZktVHGmL8AZ/E/DM9itCirWGwC4WbwN+A711kaaQYwANJfwYJMN/Uc15lU/tKCrOrFGHMnMIx3\nMDyBIe06kSoKh0E5cKvrHI3wL2BvCJfD+2nYzRjzuetMKr9pQVb1ZowZDxzLx4Q8QkSF60Qq8bpC\n1BluAFa7ztIA9wGHgKmAf2emNn3rOpPKf1qQVYMYYyZgGMocUtxNmiWuE6nEOxRWAne5zlEPEXAp\nMMJevzeya1PrruOqXsQOplWqYURkGzyew6cHx/L/7d17tJ3zncfx92fvQxOtKBKpUUUwSQgyWiXF\nIKWYjsa1o6gSXXNT7QytS2tRulSspl0WZpQWbV1C3e+X1ZYaDCY1iAiNywRNgiTITRJ7P9/54/ec\n2Nn2ybntc/Y++3xeaz1r7/08v/17vs9Zyfme3/P8LkVGNzoia2kXwfBF8DowpNGxdGA58HWIW9LH\nU4Gfhn/BWje4hWw9EhEvkLEzJe5mGvAQ7oFtfefLae7nKxsdRwfmkZ4X3wYrgUMjYqqTsXWXE7L1\nWEQsJjgEOIuHCK4nwwvHWV/YGtgAziMNhWomDwPjoTQDFmZpWNNtjY7JBiYnZOuViMgi4kfAQcxm\nOZdTaomZHKzpxP5pKs1rGh1ILiOtSrU3xEJ4vATjI+KpBodlA5ifIVvdSNo2f668NYdSZGyjI7JW\no6mw5VKYTVqmsVEWAMdAdn9q1JwH/DAiPBjQesUtZKubiJhNxucocTs3AH/Az5WtrmI/eBW4sYEx\nPAbsCKXfpc7fB0bEmU7GVg9OyFZXEbGE4HDgDB4muI6M5Y2OylrGTlAYCufQ/3/rtS928bcQb8Of\nyrBjRNzX23olZZLK+Wv1VpZ0lqQtqvYvlPSQpD06qHMzSSslPbuWcy6XtHnV/lslXVnxebikSyXN\nkbRC0jxJ90qaUFHm/2rE/Vonx8uSTs2Pd+vaWpkTstVdPgf2FOBAXmYJl1DC69pYnWT7wAvAHf14\nzqvPBQAAAAvNSURBVHeASZB9FyjD1HzpxNfrVP2ngE3z138D3gNGVuyfmpcLYGK+f0/SI/W7JI2o\nUedxwA3AMEm7dHDeAM7tJLZbgJ2ArwPbAgeRxlRsXFXPmXlc7dvfdHJ8U+DiqjJdvbaW5YRsfSYi\n7icYy/vcyw3ATQSeIsF66/NQWDe1kvujB8x9wDgo3ZMmC/tKRJwaEXXr7B0Rb7VvpGQcEfF2xf72\ne0wCFuX7nic9ux4G7Fqj2uOBq4HrgG92cOpLgGMkbVfroKQNgD2A0yLi4Yh4PSKmR8QFEXFXVfGl\nldcREQs7Of5WRLxfebpuXFvLckK2PhUR8wgmAccwM28tP9/oqGygy/aEp4EH+vAc7wDHQRwIvAl/\nzG9R39mHp+wySUNJSTeAVVXHJgJDgd8B1wJH5uWrPQrcBVzQwWmW5tvBktatU+idWtu1tTonZOtz\n+S3sawlGs4J7+C1wo1vL1gu7Q3Gdzu+39tTtwGgoXQPLgBPKsF9EzOmj03XHY5KWkBLlycB00mrl\nlSYD0/L/dzOBl4EjOqjv+8ABknavPhARZeAb+faupEcknSdphxr1XCBpSb4tlvSttRxvL1N9zq5c\nW0tzQrZ+ExHzCQ4GjuJ5FnMJJWY2OiobkApQnpB6PD9cx2rfBo6EOBhYCA+UYUxEXNlEs259FRgP\nHEoa/XV8njiB1beZDyW1jNtdSwe3rSNiFvAbYEoHx28F/or07PheYC/gKUnHVhX9CelZ8055fL9Z\ny/H2MtO7c22DQVujA7DBJf/FNk3Sg6zg59zIJGYSfBnx8UZHZwPK3lB8FM4p974ZFcBvgX+B8uLU\nQjsxg+uaKBG3eyNfU/llSesAt0navuKZ9tGk6b6fkKR8nwBJ2iYiXqpR59nAi5Im1TphRKwi/Yh/\nD5wn6RekR/iVSXdBJ8tLdna8K9fW8txCtobIW8uHAF/jBd7jYsrMpH966VhrKED5c2m4+5O9qOY1\n4BDIjgTeg9vLMDoirm3CZLxGPBFxE1AC/rVi92RSr+zxfNga3RH4r/zYRyuNeIPUwevHdG2+lVlQ\n9z+fu3JtLc8J2Romf8Z1PRljWMkd3Aj8moy5jY7MBoz9oK0AP+rBV5eSxuJsC9ndsAg4ohxxWES8\nWd8g60Y19l0EnCFpiKTxwM7ALyPi+coNuB44TlJHv/OnkG5N77v6ZNJGkn4v6WhJO0jaUtIRwPeA\n7s7Xvb6kkVXb+l29tm6ea8ByQraGi4g3CQ4DJvEar3A5cDPBu42OzJpeG5TGp67CNWfAqKFMWjVq\nFJTOhw9WwZQSjMpbZc2sVov916RHj98itYCfi4g/1yh3KzAC+LtadUXEO6Te1h+rOLYUeJw0NvqP\nwAzSrerLgJM6iavauaSxxZVbZe/uzq5tUPBc1tZUJLUBkylwHrARu1FgT9IgDrNaVkHb+XBIpOfA\na/MQ8G0oz4BiAW7I0hjbZug9beaEbM1J0ieAkxGnsy7rshdFdgHWaXRk1pRuBs1IDzdH1zj8EvA9\nyG6DQhtML8G3I+K/+zlKs7VyQramJulTwFnAPzGMjH1pYxx+2GJrWgFtU1IX419V7H6LdF/0onRL\ndH4JvgtcHxFe9sSajhOyDQiSxiCmEExiJGX2p8ioRkdlTWUaFF5Ms2AMIQ18/Q/ISrCinDot/bRi\nGkqzpuOEbAOKpD0o8DMydmErMr5AgW2o3UfTBpelUJwK2wEvQlaG98vwM+DCiFjU6PDMOuOEbANO\nPuHBoRQ4k4zxbEyJCbSxI9BvM+5aU1lAmpn5aaItWFVKd6ovzHsPmw0ITsg2YOWJeQ/EyQST+BgZ\nu1Dk86R1Yqz1vQE8QvACosACMn4CXBYR7zU6NLPuckK2liBpFHAS4h+BoYwDdkNs1uDArP5KpJmO\nH6fMHIoUeIWM84GrI2Jlg6Mz6zEnZGspkoaRxjGfTMbmbE6ZCRQZg3tmD3TzSWsuPk2JFbRR4H/I\nmALcPtgWIbDW5IRsLUlSEZhEgVPI+ALDKLFrPmRqg0ZHZ122HHgOeIoS82mjwCIyrgJ+FRHPNTg6\ns7pyQraWJ+mzwL8jvkqwDp+mzA4U2Q5Yv7NvW7/LSGOXniaYRZARiLsJrgTuGUyr/9jg4oRsg0a+\nVuxXEEcR7AcU2SJPzmOp//o11j0LSbekn6LEMtoo8CIZvwCuaeIFH8zqxgnZBiVJGwGHIL5GsA9C\nbEUwjgJj8dzZ/SED5pHmtfwzZf5CEbGU4GrgKmB6Ey6BaNZnnJBt0JM0AjiMAkeRsQcFYOs8OW8L\nrNfgAFvJMtLt6JeA2ZR4nzbEcoIHSGtD3BYR7zc0RrMGcUI2qyBpU+DwPDnvBsBISmxNG1sBW+DJ\nR7ojA/5Ceyu4xDzaACgwk4w7gfuAx/xc2MwJ2axDkjYjLdj+RQrsT8YmFAg2I2MURT4DfJq0gqwl\nAbwLzAFeIphNmZW0IZYQ3AfcC9wfEXMbGqdZE3JCNuuCfFaw0cAXgX0pMJGMYQgYQYktaOMzwOak\nYVWDYW7tjNQRa97qLWMeGStXt4KfIeMuUhJ+IiJKDYvVbABwQjbrAUkFUoLeHdidAnuTsSUA61Jm\nOMEI2hgOq7cNIU9VA0+JtJZhe/KdS4m3KFDKp1spMJeMJ4D/zbcnI+KtRoVrNhA5IZvVSd45bAJp\nwaExFNieYAzBJ1IBgk9SYhPaGI7YmA+T9VAa26peBSwGllS8tr9fQIkFFAkEZBR5hTJPkhLvU8DT\nXk3JrPeckM36UH6rexNgTL6NRoylwDjKbEZ7Gi4QDKHEEGAoYj3aGEpK1EPy1+r3RaBMunXc/trZ\n+5V8mGgXk/EeZZZQ4AOKawbOMgrMI+M1glf4sOX7bEQs64Mfldmg54Rs1iCShgLbkBL1SNJN7Q2B\njYANKTACsTHBhgQbEHXp312mwNvAG2TMIfWBnpu/tm9zI2JpHc5lZt3ghGw2QEgawocJeyPSE+kP\nSE94P+ji+5URkfV78GbWKSdkMzOzJuAF6czMzJqAE7KZmVkTcEI2MzNrAk7IVheSRkq6WNLLklZI\nmiPpDkkTq8qdIakk6ZQadRQknS5plqTlkhZKelzS5IoyV0m6pcZ395KUSRrWhVjXKFvxeUY+TKmy\n7DuSju3Oz8LMrCeckK3XJG1BmiBib+AUYBxwAPAgcElV8eOBC4DJfNQPge8APwDG5vVdBnyyi6F0\np4dirbKjACdfM2uIgTqRnzWXS0nTTuwSESsq9s+SdEX7B0l7kaa2OAv4hqTdIuLxivIHAf8ZEZUt\n4Bl9GHe1i4FzJV3n1YfMrL+5hWy9ImlDYH/gkqpkDEBELK74OBmYFhFlYBrwzari84GJkob3JJQe\nfKdSABeS/kg9qZd1mZl1mxOy9dY2pGT44toKSVofOBy4Ot91DXCEpPUqip0MjADmS3pG0qWSDqhR\n3UGSllRuwD29vhJYDpwDfD+P18ys3zghW291tWV6FPBSRDwHEBHPAK8B/9BeICJmRcQ4YFfgClJy\nvlPS5VV1/QHYEdipYqtubffUFaRFBU+rU31mZl3ihGy9NZt0u3dMJ+UmA9tL+qB9I3Xc+kjnroj4\nU0RcFBGHA8cBJ+Qdx9oti4hXI+KV9o00B3Ov5bfTfwB8R9Km9ajTzKwrnJCtVyLiHeB+4MR8sYQ1\nSNpA0jjgs8BerNmq3QeYIOmv13KKWfnrx+sa+FpExE3ATOBsutdz28ysx9zL2urhROAR4ElJZwPP\nkv5tfQn4Z1LCfjIiHq3+oqTpwAnAaZJuBB4FHiN18BoF/Jj0fPqFLsTRnY5d1WWrP5+Rx+2EbGb9\nwi1k67WIeBXYmTTueCppqNIDpIR8KnA0cFMHX78ZOFZSkZQA/x64g5SErwKeB/bv4gpFvRmHvMbn\niHiQ9Kzaf7SaWb/wak9mZmZNwC1kMzOzJuCEbC1H0j3V45TzbbGk0xsdn5lZLb5lbS0nH670kR7f\nuUUR8W5/xmNm1hVOyGZmZk3At6zNzMyagBOymZlZE3BCNjMzawJOyGZmZk3ACdnMzKwJOCGbmZk1\nASdkMzOzJvD/IGso7EAGgoIAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f1c47044d50>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "fig1, ax1 = plt.subplots()\n",
    "ax1.pie(sizes, labels=labels, startangle=90)\n",
    "ax1.axis('equal')  # Equal aspect ratio ensures that pie is drawn as a circle.\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Next we will look at the transaction amounts."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Statistics of all transaction amounts:\n",
      "+-------+------------------+\n",
      "|summary|            amount|\n",
      "+-------+------------------+\n",
      "|  count|           6362620|\n",
      "|   mean|179861.90355800136|\n",
      "| stddev| 603858.2316772807|\n",
      "|    min|               0.0|\n",
      "|    max|        9.244552E7|\n",
      "+-------+------------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print('Statistics of all transaction amounts:')\n",
    "transactionsDF.describe('amount').show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Statistics of fraudulent transaction amounts:\n",
      "+-------+-----------------+\n",
      "|summary|           amount|\n",
      "+-------+-----------------+\n",
      "|  count|             8213|\n",
      "|   mean|1467967.299835128|\n",
      "| stddev|2404252.947526678|\n",
      "|    min|              0.0|\n",
      "|    max|            1.0E7|\n",
      "+-------+-----------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "fraudulentTransactions = transactionsDF.filter(transactionsDF.isFraud == 1)\n",
    "print('Statistics of fraudulent transaction amounts:')\n",
    "fraudulentTransactions.describe('amount').show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### **3.2 Number and percentage of fraudulent transactions**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The total number of fraudulent records =  8213\n",
      "The percentage of fraudulent records = 0.129082%\n"
     ]
    }
   ],
   "source": [
    "fraudulentTransactionsCount = fraudulentTransactions.count()\n",
    "print \"The total number of fraudulent records = \", fraudulentTransactionsCount\n",
    "print \"The percentage of fraudulent records = %f%%\" % ( float(fraudulentTransactions.count())/transactionsDF.count()*100)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### **3.3 Types of the fraudulent transactions**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------+-----+\n",
      "|    type|count|\n",
      "+--------+-----+\n",
      "|CASH_OUT| 4116|\n",
      "|TRANSFER| 4097|\n",
      "+--------+-----+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "fraudulentTransactions.groupBy(fraudulentTransactions.type).count().show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can see there are two fradulent types of transactions, `CASH_OUT` and `TRANSFER`."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### **3.4 What determines whether the feature `isFlaggedFraud` gets set or not?**\n",
    "\n",
    "In the overview of the dataset, `isFlaggedFraud` is defined as: \"An illegal attempt in this dataset is an attempt to transfer more than 200,000 in a single transaction.\" Now, we will check if the dataset agrees to the description given."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The transaction types available are CASH-IN, CASH-OUT, DEBIT, PAYMENT and TRANSFER.\n",
      "\n",
      "The type of transactions in which isFlaggedFraud is set:\n",
      "\n",
      "TRANSFER\n",
      "\n"
     ]
    }
   ],
   "source": [
    "dfFlagged = transactionsDF.filter(transactionsDF.isFlaggedFraud == 1)\n",
    "flaggedTypes = dfFlagged.map(lambda x: x.type).distinct()\n",
    "\n",
    "print('The transaction types available are CASH-IN, CASH-OUT, DEBIT, PAYMENT and TRANSFER.\\n')\n",
    "print('The type of transactions in which isFlaggedFraud is set:\\n')\n",
    "for t in flaggedTypes.collect():\n",
    "    print(t + '\\n')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Out of `TRANSFER`, `CASH_IN`, `DEBIT`, `PAYMENT`, and `CASH_OUT`, only `TRANSFER` type gets the `isFlaggedFraud` set as 1 (true). This still agrees with the description in the dataset overview. Now, we will check if there is any transfer with amount more than 200,000 has `isFlaggedFraud` set."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The number of transfers not flagged: 532893\n",
      "The number of transfers with amount more than 200,000 but not flagged: 532893\n"
     ]
    }
   ],
   "source": [
    "dfTransferNotFlagged = transactionsDF.filter(transactionsDF.type == 'TRANSFER').filter(transactionsDF.isFlaggedFraud == 0)\n",
    "print('The number of transfers not flagged: ' + str(dfTransferNotFlagged.count()))\n",
    "print('The number of transfers with amount more than 200,000 but not flagged: ' + str(dfTransferNotFlagged.select(dfTransferNotFlagged.amount > 200000).count()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This result means all the non-flagged transfers are more than 200,000 and thus contradicts with the description that says any money transfer above 200,000 is flagged as fraud. Also, we already have another column `isFraud` which is the actual value whether or not the transaction is a fraud. This value should be the only benchmark and `isFlaggedFraud` is redundant, so we will drop this column."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### **3.5 Is there any correlation between fraudulent records and the originator of the transaction?**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "fraudOriginators = fraudulentTransactions.map(lambda x: (x.nameOrig, 1)).reduceByKey(lambda a, b: a + b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "How many originators had fraudulent transactions more than once? 0\n"
     ]
    }
   ],
   "source": [
    "print(\"How many originators had fraudulent transactions more than once? \" + \n",
    "      str(fraudOriginators.filter(lambda (x, y): y > 1).count()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "None of the originators had fraudulent transaction more than once, which means that this data cannot indicate the credibility of any of the originator."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### **3.6 Is there any correlation between fraudulent records and the recipient of the transaction?**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "fraudRecipients = fraudulentTransactions.map(lambda x: (x.nameDest, 1)).reduceByKey(lambda a, b: a + b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "How many recipients had fraudulent transactions more than once? 44\n",
      "The percentage of recipients that get involved in fraudulent transaction more than once: 0.538621618313%\n"
     ]
    }
   ],
   "source": [
    "secondFraudRecipient = fraudRecipients.filter(lambda (x, y): y > 1)\n",
    "noOfSecondFraudRecipient = secondFraudRecipient.count()\n",
    "\n",
    "print(\"How many recipients had fraudulent transactions more than once? \" + \n",
    "      str(noOfSecondFraudRecipient))\n",
    "\n",
    "print(\"The percentage of recipients that get involved in fraudulent transaction more than once: \" +\n",
    "     str(noOfSecondFraudRecipient * 100.0 / fraudRecipients.count()) + \"%\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Although some recipients are involved in more than 1 fraudulent transaction, it's only 0.5% of them. We can assume that there is no significant correlation between transaction destination and whether or not a transaction is fraudulent. Hence the column `nameDest` does not have much meaning."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### **3.7 Is there any common features among the fraudulent transactions?**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+----+--------+---------+-----------+--------------+--------------+-----------+--------------+--------------+-------+--------------+\n",
      "|step|    type|   amount|   nameOrig|oldbalanceOrig|newbalanceOrig|   nameDest|oldbalanceDest|newbalanceDest|isFraud|isFlaggedFraud|\n",
      "+----+--------+---------+-----------+--------------+--------------+-----------+--------------+--------------+-------+--------------+\n",
      "|   1|TRANSFER|    181.0|C1305486145|         181.0|           0.0| C553264065|           0.0|           0.0|      1|             0|\n",
      "|   1|CASH_OUT|    181.0| C840083671|         181.0|           0.0|  C38997010|       21182.0|           0.0|      1|             0|\n",
      "|   1|TRANSFER|   2806.0|C1420196421|        2806.0|           0.0| C972765878|           0.0|           0.0|      1|             0|\n",
      "|   1|CASH_OUT|   2806.0|C2101527076|        2806.0|           0.0|C1007251739|       26202.0|           0.0|      1|             0|\n",
      "|   1|TRANSFER|  20128.0| C137533655|       20128.0|           0.0|C1848415041|           0.0|           0.0|      1|             0|\n",
      "|   1|CASH_OUT|  20128.0|C1118430673|       20128.0|           0.0| C339924917|        6268.0|      12145.85|      1|             0|\n",
      "|   1|CASH_OUT|416001.34| C749981943|           0.0|           0.0| C667346055|         102.0|     9291620.0|      1|             0|\n",
      "|   1|TRANSFER|1277212.8|C1334405552|     1277212.8|           0.0| C431687661|           0.0|           0.0|      1|             0|\n",
      "|   1|CASH_OUT|1277212.8| C467632528|     1277212.8|           0.0| C716083600|           0.0|     2444985.2|      1|             0|\n",
      "|   1|TRANSFER| 35063.63|C1364127192|      35063.63|           0.0|C1136419747|           0.0|           0.0|      1|             0|\n",
      "|   1|CASH_OUT| 35063.63|C1635772897|      35063.63|           0.0|C1983025922|       31140.0|       7550.03|      1|             0|\n",
      "|   1|TRANSFER| 25071.46| C669700766|      25071.46|           0.0|C1384210339|           0.0|           0.0|      1|             0|\n",
      "|   1|CASH_OUT| 25071.46|C1275464847|      25071.46|           0.0|C1364913072|       9083.76|      34155.22|      1|             0|\n",
      "|   1|CASH_OUT|132842.64|  C13692003|       4499.08|           0.0| C297927961|           0.0|     132842.64|      1|             0|\n",
      "|   1|TRANSFER|235238.66|C1872047468|     235238.66|           0.0| C116289363|           0.0|           0.0|      1|             0|\n",
      "|   1|CASH_OUT|235238.66|C1499825229|     235238.66|           0.0|C2100440237|           0.0|     235238.66|      1|             0|\n",
      "|   2|TRANSFER|1096187.2|C1093223281|     1096187.2|           0.0|C2063275841|           0.0|           0.0|      1|             0|\n",
      "|   2|CASH_OUT|1096187.2|  C77163673|     1096187.2|           0.0| C644345897|           0.0|     1096187.2|      1|             0|\n",
      "|   2|TRANSFER| 963532.1|C1440057381|      963532.1|           0.0| C268086000|           0.0|           0.0|      1|             0|\n",
      "|   2|CASH_OUT| 963532.1| C430329518|      963532.1|           0.0| C991505714|     132382.56|     1095914.8|      1|             0|\n",
      "+----+--------+---------+-----------+--------------+--------------+-----------+--------------+--------------+-------+--------------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "fraudulentTransactions.show(20)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "It seems that the the fraud attempts are done by transferring or cashing out all of the money in the origin account. Let's check whether this feature is exclusively appearing in fraudulent transactions."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "amtEqualsOldBal = (transactionsDF\n",
    "                   .filter((transactionsDF.type == 'CASH_OUT') | (transactionsDF.type == 'TRANSFER'))\n",
    "                   .filter(transactionsDF.amount == transactionsDF.oldbalanceOrig))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total number of transactions that attempt to empty the origin account: 8034\n",
      "Of which, 100.000000% is positive for fraud\n"
     ]
    }
   ],
   "source": [
    "amtEqualsOldBalCount = amtEqualsOldBal.count()\n",
    "positiveCaseCount = (amtEqualsOldBal\n",
    "                     .filter(amtEqualsOldBal.isFraud == 1)\n",
    "                     .count())\n",
    "\n",
    "print \"Total number of transactions that attempt to empty the origin account: %d\" % (amtEqualsOldBalCount)\n",
    "print \"Of which, %f%% is positive for fraud\" % (float(positiveCaseCount) * 100 / amtEqualsOldBalCount)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "It turned out that this feature is closely related to fradulent transactions."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### **Result**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In 3.3, we saw that fraudulent transactions only take form in 'CASH_OUT' and 'TRANSFER'. Therefore, we can skip checking other type of transactions."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Size of new dataset:  2770409\n"
     ]
    }
   ],
   "source": [
    "newTransactionsDF = transactionsDF.filter((transactionsDF.type == 'CASH_OUT') | (transactionsDF.type == 'TRANSFER'))\n",
    "print 'Size of new dataset: ', newTransactionsDF.count()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "From section 3.4 - 3.6, we see that the columns `isFlaggedFraud`, `nameOrig` and `nameDest` do not help in predicting fraudulent transactions. Therefore, we will not use those columns for training our model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "newTransactionsDF = newTransactionsDF.drop('isFlaggedFraud').drop('nameOrig').drop('nameDest')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Lastly, we found in 3.7 that fraudulent transactions often attempt to empty the origin account by transferring or cashing out the full account balance. Based on this information, we will append a new column which indicates whether a transaction has this feature."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+----+--------+---------+--------------+--------------+--------------+--------------+-------+--------------+\n",
      "|step|    type|   amount|oldbalanceOrig|newbalanceOrig|oldbalanceDest|newbalanceDest|isFraud|isEmptyAttempt|\n",
      "+----+--------+---------+--------------+--------------+--------------+--------------+-------+--------------+\n",
      "|   1|TRANSFER|    181.0|         181.0|           0.0|           0.0|           0.0|      1|          true|\n",
      "|   1|CASH_OUT|    181.0|         181.0|           0.0|       21182.0|           0.0|      1|          true|\n",
      "|   1|CASH_OUT|229133.94|       15325.0|           0.0|        5083.0|      51513.44|      0|         false|\n",
      "|   1|TRANSFER| 215310.3|         705.0|           0.0|       22425.0|           0.0|      0|         false|\n",
      "|   1|TRANSFER|311685.88|       10835.0|           0.0|        6267.0|     2719173.0|      0|         false|\n",
      "+----+--------+---------+--------------+--------------+--------------+--------------+-------+--------------+\n",
      "only showing top 5 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "newTransactionsDF = (newTransactionsDF\n",
    "                     .withColumn('isEmptyAttempt', (newTransactionsDF.amount == newTransactionsDF.oldbalanceOrig)))\n",
    "newTransactionsDF.show(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Since we will not use transactionsDF anymore, we can uncache it."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DataFrame[step: int, type: string, amount: float, nameOrig: string, oldbalanceOrig: float, newbalanceOrig: float, nameDest: string, oldbalanceDest: float, newbalanceDest: float, isFraud: int, isFlaggedFraud: int]"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "transactionsDF.unpersist()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Part 4: Exploring classification algorithms"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### **4.1 Transforming the data**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To use the machine learning algorithms provided by MlLib, we have to do a few transformations on our input data:\n",
    "1. Convert the label column (i.e., 'isFraud') to doubleType.\n",
    "2. Convert the categorical attributes into numeric attributes. The only categorical attribute that we will use is 'type'.\n",
    "3. Combine the values of all feature/attribute columns into a feature vector.\n",
    "4. Pass the feature vector through vector indexer to differentiate between categorical and continuous attribute."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from pyspark.ml import Pipeline\n",
    "from pyspark.ml.classification import RandomForestClassifier\n",
    "from pyspark.ml.feature import StringIndexer, VectorIndexer\n",
    "from pyspark.ml.feature import VectorAssembler\n",
    "from pyspark.ml.evaluation import MulticlassClassificationEvaluator\n",
    "\n",
    "\n",
    "# Cast class column 'isFraud' from intType to doubleType\n",
    "labelIndexer = StringIndexer(inputCol=\"isFraud\", outputCol=\"label\")\n",
    "\n",
    "# Convert categorical column 'type' to numerical index\n",
    "typeIndexer = StringIndexer(inputCol=\"type\", outputCol=\"typeIndex\")\n",
    "\n",
    "# Specify the feature columns - a parameter for featureAssembler\n",
    "featuresCols = newTransactionsDF.columns\n",
    "featuresCols.remove('isFraud') \n",
    "featuresCols.remove('type')\n",
    "\n",
    "featureAssembler = VectorAssembler(inputCols=featuresCols, outputCol=\"featuresVec\")\n",
    "featureIndexer = VectorIndexer(inputCol=\"featuresVec\", outputCol=\"features\", maxCategories=5)\n",
    "\n",
    "pipeline = Pipeline(stages=[labelIndexer, typeIndexer, featureAssembler, featureIndexer])\n",
    "pipelineModel = pipeline.fit(newTransactionsDF)\n",
    "data = pipelineModel.transform(newTransactionsDF)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+----+--------+---------+--------------+--------------+--------------+--------------+-------+--------------+-----+---------+--------------------+--------------------+\n",
      "|step|    type|   amount|oldbalanceOrig|newbalanceOrig|oldbalanceDest|newbalanceDest|isFraud|isEmptyAttempt|label|typeIndex|         featuresVec|            features|\n",
      "+----+--------+---------+--------------+--------------+--------------+--------------+-------+--------------+-----+---------+--------------------+--------------------+\n",
      "|   1|TRANSFER|    181.0|         181.0|           0.0|           0.0|           0.0|      1|          true|  1.0|      1.0|[1.0,181.0,181.0,...|[1.0,181.0,181.0,...|\n",
      "|   1|CASH_OUT|    181.0|         181.0|           0.0|       21182.0|           0.0|      1|          true|  1.0|      0.0|[1.0,181.0,181.0,...|[1.0,181.0,181.0,...|\n",
      "|   1|CASH_OUT|229133.94|       15325.0|           0.0|        5083.0|      51513.44|      0|         false|  0.0|      0.0|[1.0,229133.9375,...|[1.0,229133.9375,...|\n",
      "|   1|TRANSFER| 215310.3|         705.0|           0.0|       22425.0|           0.0|      0|         false|  0.0|      1.0|[1.0,215310.29687...|[1.0,215310.29687...|\n",
      "|   1|TRANSFER|311685.88|       10835.0|           0.0|        6267.0|     2719173.0|      0|         false|  0.0|      1.0|[1.0,311685.875,1...|[1.0,311685.875,1...|\n",
      "+----+--------+---------+--------------+--------------+--------------+--------------+-------+--------------+-----+---------+--------------------+--------------------+\n",
      "only showing top 5 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# To display the table in a better format, you may zoom in/out to adjust\n",
    "data.show(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### **4.2 Splitting the data**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We need to divide our transformed input data into two sets:\n",
    "1. **Training set**: for training the models\n",
    "2. **Testing set**: for evaluating the models\n",
    "\n",
    "Here we will split the data randomly with proportion of 70% training set and 30% testing set."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "(trainingData, testingData) = data.randomSplit([0.7, 0.3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DataFrame[step: int, type: string, amount: float, oldbalanceOrig: float, newbalanceOrig: float, oldbalanceDest: float, newbalanceDest: float, isFraud: int, isEmptyAttempt: boolean, label: double, typeIndex: double, featuresVec: vector, features: vector]"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainingData.cache()\n",
    "testingData.cache()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### **4.3 Exploring the classifier models**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In this part, we are going to explore two classifiers that are supported by MlLib: Random Forest and Logistic Regression.\n",
    "\n",
    "After obtaining the trained models, we will use them to make predictions on the testing data and evaluate their performance using BinaryClassificationEvaluator. \n",
    "\n",
    "The BinaryClassificationEvaluator supports two evaluation metric: areaUnderROC(recall-specifity) and areaUnderPR(precision-recall). Since we are more interested in the positive class which denotes fraud, we are going to use areaUnderPR."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**1. Random Forest Classifier**\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Train the model\n",
    "rf = RandomForestClassifier(labelCol=\"label\", featuresCol=\"features\")\n",
    "rfModel = rf.fit(trainingData)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- step: integer (nullable = true)\n",
      " |-- type: string (nullable = true)\n",
      " |-- amount: float (nullable = true)\n",
      " |-- oldbalanceOrig: float (nullable = true)\n",
      " |-- newbalanceOrig: float (nullable = true)\n",
      " |-- oldbalanceDest: float (nullable = true)\n",
      " |-- newbalanceDest: float (nullable = true)\n",
      " |-- isFraud: integer (nullable = true)\n",
      " |-- isEmptyAttempt: boolean (nullable = true)\n",
      " |-- label: double (nullable = true)\n",
      " |-- typeIndex: double (nullable = true)\n",
      " |-- featuresVec: vector (nullable = true)\n",
      " |-- features: vector (nullable = true)\n",
      " |-- rawPrediction: vector (nullable = true)\n",
      " |-- probability: vector (nullable = true)\n",
      " |-- prediction: double (nullable = true)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Make predictons on test data using the trained model\n",
    "rfPredictions = rfModel.transform(testingData)\n",
    "rfPredictions.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Score for Random Forest model = 0.997265\n"
     ]
    }
   ],
   "source": [
    "from pyspark.ml.evaluation import BinaryClassificationEvaluator \n",
    "\n",
    "# Evaluate model\n",
    "evaluator = BinaryClassificationEvaluator(labelCol=\"label\", rawPredictionCol=\"rawPrediction\", metricName=\"areaUnderPR\")\n",
    "rfScore = evaluator.evaluate(rfPredictions)\n",
    "print(\"Score for Random Forest model = %g\" % rfScore)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**2. Logistic Regression with weights**\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def generateClassWeight(df):\n",
    "    positivesCount = df.filter(df.isFraud == 1).count()\n",
    "    totalCount = df.count()\n",
    "    weight = (totalCount - positivesCount) / float(totalCount)\n",
    "    return df.withColumn('classWeight', when(df.isFraud == 0, weight).otherwise(1.0 - weight))\n",
    "\n",
    "weightedTrainingData = generateClassWeight(trainingData)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/spark/python/pyspark/ml/classification.py:207: UserWarning: weights is deprecated. Use coefficients instead.\n",
      "  warnings.warn(\"weights is deprecated. Use coefficients instead.\")\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Score for weighted logistic regression model = 0.995731\n"
     ]
    }
   ],
   "source": [
    "from pyspark.ml.classification import LogisticRegression\n",
    "\n",
    "# Training\n",
    "lrWeighted = LogisticRegression(labelCol=\"label\", featuresCol=\"features\").setWeightCol(\"classWeight\")\n",
    "lrWeightedModel = lrWeighted.fit(weightedTrainingData)\n",
    "\n",
    "# Testing\n",
    "lrWeightedPredictions = lrWeightedModel.transform(testingData)\n",
    "\n",
    "# Evaluation\n",
    "lrWeightedScore = evaluator.evaluate(lrWeightedPredictions)\n",
    "print(\"Score for weighted logistic regression model = %g\" % lrWeightedScore)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**3. Logistic Regression without weights**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Score for logistic regression model = 0.989799\n"
     ]
    }
   ],
   "source": [
    "# Training\n",
    "lr = LogisticRegression(labelCol=\"label\", featuresCol=\"features\")\n",
    "lrModel = lr.fit(trainingData)\n",
    "\n",
    "# Testing\n",
    "lrPredictions = lrModel.transform(testingData)\n",
    "\n",
    "# Evaluation\n",
    "lrScore = evaluator.evaluate(lrPredictions)\n",
    "print(\"Score for logistic regression model = %g\" % lrScore)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Part 5: Conclusion"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Comparing the three classifiers, we conclude that Random Forest is the best model among three as it generates the highest metric score.\n",
    "In the future, it is possible to further test on other models, and adjust some hyperparameters to fine tune the model."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "Since ML pipeline persistence is not supported in Spark 1.6 using python, we are not able to save the model.\n",
    "However, if Spark 2.0 or later version is used, it is possible to save the model in Amazon S3 and load it for further use:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Train the model with the whole processed data\n",
    "# rfModel_all = rf.fit(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Save the model\n",
    "# rfModel_all.save(sc, \"s3n://%s:%s@%s/%s\" % (ACCESS_KEY, SECRET_KEY, AWS_BUCKET_NAME, \"predicition_model\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Load the model\n",
    "# trained_rfModel = RandomForestClassifier.load(sc.textFile(\"s3n://%s:%s@%s/%s\" % (ACCESS_KEY, SECRET_KEY, AWS_BUCKET_NAME, predicition_model))"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}