1. UNSW-NB15 - Ohe and frequency.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "45ca8d64",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((175341, 45), (82332, 45))"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "train = pd.read_csv('UNSW_NB15_training-set.csv')\n",
    "test = pd.read_csv('UNSW_NB15_testing-set.csv')\n",
    "train.shape, test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "d81c96ef",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(257673, 46)"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Combine into file - 'concat_data':\n",
    "train['source']= 'train'\n",
    "test['source'] = 'test'\n",
    "concat_data=pd.concat([train, test],ignore_index=True)\n",
    "concat_data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "7e4fdc41",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>dur</th>\n",
       "      <th>proto</th>\n",
       "      <th>service</th>\n",
       "      <th>state</th>\n",
       "      <th>spkts</th>\n",
       "      <th>dpkts</th>\n",
       "      <th>sbytes</th>\n",
       "      <th>dbytes</th>\n",
       "      <th>rate</th>\n",
       "      <th>...</th>\n",
       "      <th>ct_dst_src_ltm</th>\n",
       "      <th>is_ftp_login</th>\n",
       "      <th>ct_ftp_cmd</th>\n",
       "      <th>ct_flw_http_mthd</th>\n",
       "      <th>ct_src_ltm</th>\n",
       "      <th>ct_srv_dst</th>\n",
       "      <th>is_sm_ips_ports</th>\n",
       "      <th>attack_cat</th>\n",
       "      <th>label</th>\n",
       "      <th>source</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0.121478</td>\n",
       "      <td>tcp</td>\n",
       "      <td>-</td>\n",
       "      <td>FIN</td>\n",
       "      <td>6</td>\n",
       "      <td>4</td>\n",
       "      <td>258</td>\n",
       "      <td>172</td>\n",
       "      <td>74.087490</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>Normal</td>\n",
       "      <td>0</td>\n",
       "      <td>train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>0.649902</td>\n",
       "      <td>tcp</td>\n",
       "      <td>-</td>\n",
       "      <td>FIN</td>\n",
       "      <td>14</td>\n",
       "      <td>38</td>\n",
       "      <td>734</td>\n",
       "      <td>42014</td>\n",
       "      <td>78.473372</td>\n",
       "      <td>...</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>Normal</td>\n",
       "      <td>0</td>\n",
       "      <td>train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>1.623129</td>\n",
       "      <td>tcp</td>\n",
       "      <td>-</td>\n",
       "      <td>FIN</td>\n",
       "      <td>8</td>\n",
       "      <td>16</td>\n",
       "      <td>364</td>\n",
       "      <td>13186</td>\n",
       "      <td>14.170161</td>\n",
       "      <td>...</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>Normal</td>\n",
       "      <td>0</td>\n",
       "      <td>train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>1.681642</td>\n",
       "      <td>tcp</td>\n",
       "      <td>ftp</td>\n",
       "      <td>FIN</td>\n",
       "      <td>12</td>\n",
       "      <td>12</td>\n",
       "      <td>628</td>\n",
       "      <td>770</td>\n",
       "      <td>13.677108</td>\n",
       "      <td>...</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>Normal</td>\n",
       "      <td>0</td>\n",
       "      <td>train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>0.449454</td>\n",
       "      <td>tcp</td>\n",
       "      <td>-</td>\n",
       "      <td>FIN</td>\n",
       "      <td>10</td>\n",
       "      <td>6</td>\n",
       "      <td>534</td>\n",
       "      <td>268</td>\n",
       "      <td>33.373826</td>\n",
       "      <td>...</td>\n",
       "      <td>40</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>39</td>\n",
       "      <td>0</td>\n",
       "      <td>Normal</td>\n",
       "      <td>0</td>\n",
       "      <td>train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>257668</th>\n",
       "      <td>82328</td>\n",
       "      <td>0.000005</td>\n",
       "      <td>udp</td>\n",
       "      <td>-</td>\n",
       "      <td>INT</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>104</td>\n",
       "      <td>0</td>\n",
       "      <td>200000.005100</td>\n",
       "      <td>...</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>Normal</td>\n",
       "      <td>0</td>\n",
       "      <td>test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>257669</th>\n",
       "      <td>82329</td>\n",
       "      <td>1.106101</td>\n",
       "      <td>tcp</td>\n",
       "      <td>-</td>\n",
       "      <td>FIN</td>\n",
       "      <td>20</td>\n",
       "      <td>8</td>\n",
       "      <td>18062</td>\n",
       "      <td>354</td>\n",
       "      <td>24.410067</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>Normal</td>\n",
       "      <td>0</td>\n",
       "      <td>test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>257670</th>\n",
       "      <td>82330</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>arp</td>\n",
       "      <td>-</td>\n",
       "      <td>INT</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>46</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Normal</td>\n",
       "      <td>0</td>\n",
       "      <td>test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>257671</th>\n",
       "      <td>82331</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>arp</td>\n",
       "      <td>-</td>\n",
       "      <td>INT</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>46</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Normal</td>\n",
       "      <td>0</td>\n",
       "      <td>test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>257672</th>\n",
       "      <td>82332</td>\n",
       "      <td>0.000009</td>\n",
       "      <td>udp</td>\n",
       "      <td>-</td>\n",
       "      <td>INT</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>104</td>\n",
       "      <td>0</td>\n",
       "      <td>111111.107200</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>Normal</td>\n",
       "      <td>0</td>\n",
       "      <td>test</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>257673 rows × 46 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           id       dur proto service state  spkts  dpkts  sbytes  dbytes  \\\n",
       "0           1  0.121478   tcp       -   FIN      6      4     258     172   \n",
       "1           2  0.649902   tcp       -   FIN     14     38     734   42014   \n",
       "2           3  1.623129   tcp       -   FIN      8     16     364   13186   \n",
       "3           4  1.681642   tcp     ftp   FIN     12     12     628     770   \n",
       "4           5  0.449454   tcp       -   FIN     10      6     534     268   \n",
       "...       ...       ...   ...     ...   ...    ...    ...     ...     ...   \n",
       "257668  82328  0.000005   udp       -   INT      2      0     104       0   \n",
       "257669  82329  1.106101   tcp       -   FIN     20      8   18062     354   \n",
       "257670  82330  0.000000   arp       -   INT      1      0      46       0   \n",
       "257671  82331  0.000000   arp       -   INT      1      0      46       0   \n",
       "257672  82332  0.000009   udp       -   INT      2      0     104       0   \n",
       "\n",
       "                 rate  ...  ct_dst_src_ltm  is_ftp_login  ct_ftp_cmd  \\\n",
       "0           74.087490  ...               1             0           0   \n",
       "1           78.473372  ...               2             0           0   \n",
       "2           14.170161  ...               3             0           0   \n",
       "3           13.677108  ...               3             1           1   \n",
       "4           33.373826  ...              40             0           0   \n",
       "...               ...  ...             ...           ...         ...   \n",
       "257668  200000.005100  ...               2             0           0   \n",
       "257669      24.410067  ...               1             0           0   \n",
       "257670       0.000000  ...               1             0           0   \n",
       "257671       0.000000  ...               1             0           0   \n",
       "257672  111111.107200  ...               1             0           0   \n",
       "\n",
       "        ct_flw_http_mthd  ct_src_ltm  ct_srv_dst  is_sm_ips_ports  attack_cat  \\\n",
       "0                      0           1           1                0      Normal   \n",
       "1                      0           1           6                0      Normal   \n",
       "2                      0           2           6                0      Normal   \n",
       "3                      0           2           1                0      Normal   \n",
       "4                      0           2          39                0      Normal   \n",
       "...                  ...         ...         ...              ...         ...   \n",
       "257668                 0           2           1                0      Normal   \n",
       "257669                 0           3           2                0      Normal   \n",
       "257670                 0           1           1                1      Normal   \n",
       "257671                 0           1           1                1      Normal   \n",
       "257672                 0           1           1                0      Normal   \n",
       "\n",
       "        label  source  \n",
       "0           0   train  \n",
       "1           0   train  \n",
       "2           0   train  \n",
       "3           0   train  \n",
       "4           0   train  \n",
       "...       ...     ...  \n",
       "257668      0    test  \n",
       "257669      0    test  \n",
       "257670      0    test  \n",
       "257671      0    test  \n",
       "257672      0    test  \n",
       "\n",
       "[257673 rows x 46 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "concat_data.head(257673)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "ebf454db",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>dur</th>\n",
       "      <th>proto</th>\n",
       "      <th>service</th>\n",
       "      <th>state</th>\n",
       "      <th>spkts</th>\n",
       "      <th>dpkts</th>\n",
       "      <th>sbytes</th>\n",
       "      <th>dbytes</th>\n",
       "      <th>rate</th>\n",
       "      <th>...</th>\n",
       "      <th>ct_flw_http_mthd</th>\n",
       "      <th>ct_src_ltm</th>\n",
       "      <th>ct_srv_dst</th>\n",
       "      <th>is_sm_ips_ports</th>\n",
       "      <th>attack_cat</th>\n",
       "      <th>label</th>\n",
       "      <th>source</th>\n",
       "      <th>proto_freq_encode</th>\n",
       "      <th>service_freq_encode</th>\n",
       "      <th>state_freq_encode</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0.121478</td>\n",
       "      <td>tcp</td>\n",
       "      <td>-</td>\n",
       "      <td>FIN</td>\n",
       "      <td>6</td>\n",
       "      <td>4</td>\n",
       "      <td>258</td>\n",
       "      <td>172</td>\n",
       "      <td>74.087490</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>Normal</td>\n",
       "      <td>0</td>\n",
       "      <td>train</td>\n",
       "      <td>0.477508</td>\n",
       "      <td>0.548451</td>\n",
       "      <td>0.454700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>0.649902</td>\n",
       "      <td>tcp</td>\n",
       "      <td>-</td>\n",
       "      <td>FIN</td>\n",
       "      <td>14</td>\n",
       "      <td>38</td>\n",
       "      <td>734</td>\n",
       "      <td>42014</td>\n",
       "      <td>78.473372</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>Normal</td>\n",
       "      <td>0</td>\n",
       "      <td>train</td>\n",
       "      <td>0.477508</td>\n",
       "      <td>0.548451</td>\n",
       "      <td>0.454700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>1.623129</td>\n",
       "      <td>tcp</td>\n",
       "      <td>-</td>\n",
       "      <td>FIN</td>\n",
       "      <td>8</td>\n",
       "      <td>16</td>\n",
       "      <td>364</td>\n",
       "      <td>13186</td>\n",
       "      <td>14.170161</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>Normal</td>\n",
       "      <td>0</td>\n",
       "      <td>train</td>\n",
       "      <td>0.477508</td>\n",
       "      <td>0.548451</td>\n",
       "      <td>0.454700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>1.681642</td>\n",
       "      <td>tcp</td>\n",
       "      <td>ftp</td>\n",
       "      <td>FIN</td>\n",
       "      <td>12</td>\n",
       "      <td>12</td>\n",
       "      <td>628</td>\n",
       "      <td>770</td>\n",
       "      <td>13.677108</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>Normal</td>\n",
       "      <td>0</td>\n",
       "      <td>train</td>\n",
       "      <td>0.477508</td>\n",
       "      <td>0.019327</td>\n",
       "      <td>0.454700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>0.449454</td>\n",
       "      <td>tcp</td>\n",
       "      <td>-</td>\n",
       "      <td>FIN</td>\n",
       "      <td>10</td>\n",
       "      <td>6</td>\n",
       "      <td>534</td>\n",
       "      <td>268</td>\n",
       "      <td>33.373826</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>39</td>\n",
       "      <td>0</td>\n",
       "      <td>Normal</td>\n",
       "      <td>0</td>\n",
       "      <td>train</td>\n",
       "      <td>0.477508</td>\n",
       "      <td>0.548451</td>\n",
       "      <td>0.454700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>257668</th>\n",
       "      <td>82328</td>\n",
       "      <td>0.000005</td>\n",
       "      <td>udp</td>\n",
       "      <td>-</td>\n",
       "      <td>INT</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>104</td>\n",
       "      <td>0</td>\n",
       "      <td>200000.005100</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>Normal</td>\n",
       "      <td>0</td>\n",
       "      <td>test</td>\n",
       "      <td>0.359762</td>\n",
       "      <td>0.548451</td>\n",
       "      <td>0.451883</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>257669</th>\n",
       "      <td>82329</td>\n",
       "      <td>1.106101</td>\n",
       "      <td>tcp</td>\n",
       "      <td>-</td>\n",
       "      <td>FIN</td>\n",
       "      <td>20</td>\n",
       "      <td>8</td>\n",
       "      <td>18062</td>\n",
       "      <td>354</td>\n",
       "      <td>24.410067</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>Normal</td>\n",
       "      <td>0</td>\n",
       "      <td>test</td>\n",
       "      <td>0.477508</td>\n",
       "      <td>0.548451</td>\n",
       "      <td>0.454700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>257670</th>\n",
       "      <td>82330</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>arp</td>\n",
       "      <td>-</td>\n",
       "      <td>INT</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>46</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Normal</td>\n",
       "      <td>0</td>\n",
       "      <td>test</td>\n",
       "      <td>0.014926</td>\n",
       "      <td>0.548451</td>\n",
       "      <td>0.451883</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>257671</th>\n",
       "      <td>82331</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>arp</td>\n",
       "      <td>-</td>\n",
       "      <td>INT</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>46</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Normal</td>\n",
       "      <td>0</td>\n",
       "      <td>test</td>\n",
       "      <td>0.014926</td>\n",
       "      <td>0.548451</td>\n",
       "      <td>0.451883</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>257672</th>\n",
       "      <td>82332</td>\n",
       "      <td>0.000009</td>\n",
       "      <td>udp</td>\n",
       "      <td>-</td>\n",
       "      <td>INT</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>104</td>\n",
       "      <td>0</td>\n",
       "      <td>111111.107200</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>Normal</td>\n",
       "      <td>0</td>\n",
       "      <td>test</td>\n",
       "      <td>0.359762</td>\n",
       "      <td>0.548451</td>\n",
       "      <td>0.451883</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>257673 rows × 49 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           id       dur proto service state  spkts  dpkts  sbytes  dbytes  \\\n",
       "0           1  0.121478   tcp       -   FIN      6      4     258     172   \n",
       "1           2  0.649902   tcp       -   FIN     14     38     734   42014   \n",
       "2           3  1.623129   tcp       -   FIN      8     16     364   13186   \n",
       "3           4  1.681642   tcp     ftp   FIN     12     12     628     770   \n",
       "4           5  0.449454   tcp       -   FIN     10      6     534     268   \n",
       "...       ...       ...   ...     ...   ...    ...    ...     ...     ...   \n",
       "257668  82328  0.000005   udp       -   INT      2      0     104       0   \n",
       "257669  82329  1.106101   tcp       -   FIN     20      8   18062     354   \n",
       "257670  82330  0.000000   arp       -   INT      1      0      46       0   \n",
       "257671  82331  0.000000   arp       -   INT      1      0      46       0   \n",
       "257672  82332  0.000009   udp       -   INT      2      0     104       0   \n",
       "\n",
       "                 rate  ...  ct_flw_http_mthd  ct_src_ltm  ct_srv_dst  \\\n",
       "0           74.087490  ...                 0           1           1   \n",
       "1           78.473372  ...                 0           1           6   \n",
       "2           14.170161  ...                 0           2           6   \n",
       "3           13.677108  ...                 0           2           1   \n",
       "4           33.373826  ...                 0           2          39   \n",
       "...               ...  ...               ...         ...         ...   \n",
       "257668  200000.005100  ...                 0           2           1   \n",
       "257669      24.410067  ...                 0           3           2   \n",
       "257670       0.000000  ...                 0           1           1   \n",
       "257671       0.000000  ...                 0           1           1   \n",
       "257672  111111.107200  ...                 0           1           1   \n",
       "\n",
       "        is_sm_ips_ports  attack_cat  label  source  proto_freq_encode  \\\n",
       "0                     0      Normal      0   train           0.477508   \n",
       "1                     0      Normal      0   train           0.477508   \n",
       "2                     0      Normal      0   train           0.477508   \n",
       "3                     0      Normal      0   train           0.477508   \n",
       "4                     0      Normal      0   train           0.477508   \n",
       "...                 ...         ...    ...     ...                ...   \n",
       "257668                0      Normal      0    test           0.359762   \n",
       "257669                0      Normal      0    test           0.477508   \n",
       "257670                1      Normal      0    test           0.014926   \n",
       "257671                1      Normal      0    test           0.014926   \n",
       "257672                0      Normal      0    test           0.359762   \n",
       "\n",
       "        service_freq_encode  state_freq_encode  \n",
       "0                  0.548451           0.454700  \n",
       "1                  0.548451           0.454700  \n",
       "2                  0.548451           0.454700  \n",
       "3                  0.019327           0.454700  \n",
       "4                  0.548451           0.454700  \n",
       "...                     ...                ...  \n",
       "257668             0.548451           0.451883  \n",
       "257669             0.548451           0.454700  \n",
       "257670             0.548451           0.451883  \n",
       "257671             0.548451           0.451883  \n",
       "257672             0.548451           0.451883  \n",
       "\n",
       "[257673 rows x 49 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "concat_data = pd.DataFrame(concat_data)\n",
    "fe = concat_data.groupby('proto').size()/len(concat_data)\n",
    "concat_data.loc[:,'proto_freq_encode'] = concat_data['proto'].map(fe)\n",
    "fe = concat_data.groupby('service').size()/len(concat_data)\n",
    "concat_data.loc[:,'service_freq_encode'] = concat_data['service'].map(fe)\n",
    "fe = concat_data.groupby('state').size()/len(concat_data)\n",
    "concat_data.loc[:,'state_freq_encode'] = concat_data['state'].map(fe)\n",
    "concat_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "c14f43a4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>dur</th>\n",
       "      <th>spkts</th>\n",
       "      <th>dpkts</th>\n",
       "      <th>sbytes</th>\n",
       "      <th>dbytes</th>\n",
       "      <th>rate</th>\n",
       "      <th>sttl</th>\n",
       "      <th>dttl</th>\n",
       "      <th>sload</th>\n",
       "      <th>...</th>\n",
       "      <th>state_CLO</th>\n",
       "      <th>state_CON</th>\n",
       "      <th>state_ECO</th>\n",
       "      <th>state_FIN</th>\n",
       "      <th>state_INT</th>\n",
       "      <th>state_PAR</th>\n",
       "      <th>state_REQ</th>\n",
       "      <th>state_RST</th>\n",
       "      <th>state_URN</th>\n",
       "      <th>state_no</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0.121478</td>\n",
       "      <td>6</td>\n",
       "      <td>4</td>\n",
       "      <td>258</td>\n",
       "      <td>172</td>\n",
       "      <td>74.087490</td>\n",
       "      <td>252</td>\n",
       "      <td>254</td>\n",
       "      <td>1.415894e+04</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>0.649902</td>\n",
       "      <td>14</td>\n",
       "      <td>38</td>\n",
       "      <td>734</td>\n",
       "      <td>42014</td>\n",
       "      <td>78.473372</td>\n",
       "      <td>62</td>\n",
       "      <td>252</td>\n",
       "      <td>8.395112e+03</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>1.623129</td>\n",
       "      <td>8</td>\n",
       "      <td>16</td>\n",
       "      <td>364</td>\n",
       "      <td>13186</td>\n",
       "      <td>14.170161</td>\n",
       "      <td>62</td>\n",
       "      <td>252</td>\n",
       "      <td>1.572272e+03</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>1.681642</td>\n",
       "      <td>12</td>\n",
       "      <td>12</td>\n",
       "      <td>628</td>\n",
       "      <td>770</td>\n",
       "      <td>13.677108</td>\n",
       "      <td>62</td>\n",
       "      <td>252</td>\n",
       "      <td>2.740179e+03</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>0.449454</td>\n",
       "      <td>10</td>\n",
       "      <td>6</td>\n",
       "      <td>534</td>\n",
       "      <td>268</td>\n",
       "      <td>33.373826</td>\n",
       "      <td>254</td>\n",
       "      <td>252</td>\n",
       "      <td>8.561499e+03</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>257668</th>\n",
       "      <td>82328</td>\n",
       "      <td>0.000005</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>104</td>\n",
       "      <td>0</td>\n",
       "      <td>200000.005100</td>\n",
       "      <td>254</td>\n",
       "      <td>0</td>\n",
       "      <td>8.320000e+07</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>257669</th>\n",
       "      <td>82329</td>\n",
       "      <td>1.106101</td>\n",
       "      <td>20</td>\n",
       "      <td>8</td>\n",
       "      <td>18062</td>\n",
       "      <td>354</td>\n",
       "      <td>24.410067</td>\n",
       "      <td>254</td>\n",
       "      <td>252</td>\n",
       "      <td>1.241044e+05</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>257670</th>\n",
       "      <td>82330</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>46</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>257671</th>\n",
       "      <td>82331</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>46</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>257672</th>\n",
       "      <td>82332</td>\n",
       "      <td>0.000009</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>104</td>\n",
       "      <td>0</td>\n",
       "      <td>111111.107200</td>\n",
       "      <td>254</td>\n",
       "      <td>0</td>\n",
       "      <td>4.622222e+07</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>257673 rows × 203 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           id       dur  spkts  dpkts  sbytes  dbytes           rate  sttl  \\\n",
       "0           1  0.121478      6      4     258     172      74.087490   252   \n",
       "1           2  0.649902     14     38     734   42014      78.473372    62   \n",
       "2           3  1.623129      8     16     364   13186      14.170161    62   \n",
       "3           4  1.681642     12     12     628     770      13.677108    62   \n",
       "4           5  0.449454     10      6     534     268      33.373826   254   \n",
       "...       ...       ...    ...    ...     ...     ...            ...   ...   \n",
       "257668  82328  0.000005      2      0     104       0  200000.005100   254   \n",
       "257669  82329  1.106101     20      8   18062     354      24.410067   254   \n",
       "257670  82330  0.000000      1      0      46       0       0.000000     0   \n",
       "257671  82331  0.000000      1      0      46       0       0.000000     0   \n",
       "257672  82332  0.000009      2      0     104       0  111111.107200   254   \n",
       "\n",
       "        dttl         sload  ...  state_CLO  state_CON  state_ECO  state_FIN  \\\n",
       "0        254  1.415894e+04  ...          0          0          0          1   \n",
       "1        252  8.395112e+03  ...          0          0          0          1   \n",
       "2        252  1.572272e+03  ...          0          0          0          1   \n",
       "3        252  2.740179e+03  ...          0          0          0          1   \n",
       "4        252  8.561499e+03  ...          0          0          0          1   \n",
       "...      ...           ...  ...        ...        ...        ...        ...   \n",
       "257668     0  8.320000e+07  ...          0          0          0          0   \n",
       "257669   252  1.241044e+05  ...          0          0          0          1   \n",
       "257670     0  0.000000e+00  ...          0          0          0          0   \n",
       "257671     0  0.000000e+00  ...          0          0          0          0   \n",
       "257672     0  4.622222e+07  ...          0          0          0          0   \n",
       "\n",
       "        state_INT  state_PAR  state_REQ  state_RST  state_URN  state_no  \n",
       "0               0          0          0          0          0         0  \n",
       "1               0          0          0          0          0         0  \n",
       "2               0          0          0          0          0         0  \n",
       "3               0          0          0          0          0         0  \n",
       "4               0          0          0          0          0         0  \n",
       "...           ...        ...        ...        ...        ...       ...  \n",
       "257668          1          0          0          0          0         0  \n",
       "257669          0          0          0          0          0         0  \n",
       "257670          1          0          0          0          0         0  \n",
       "257671          1          0          0          0          0         0  \n",
       "257672          1          0          0          0          0         0  \n",
       "\n",
       "[257673 rows x 203 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "var_to_encode = ['proto','service','state']\n",
    "concat_data = pd.get_dummies(concat_data, columns=var_to_encode)\n",
    "concat_data.columns\n",
    "concat_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "ccee6776",
   "metadata": {},
   "outputs": [],
   "source": [
    "concat_data.drop('attack_cat',axis = 1,inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "8133cd25",
   "metadata": {},
   "outputs": [],
   "source": [
    "train = concat_data.loc[concat_data['source']=='train']\n",
    "test = concat_data.loc[concat_data['source']=='test']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "60dae533",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\admin\\anaconda3\\lib\\site-packages\\pandas\\core\\frame.py:4308: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  return super().drop(\n"
     ]
    }
   ],
   "source": [
    "train.drop('source',axis=1,inplace=True)\n",
    "test.drop('source',axis=1,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "633afbd8",
   "metadata": {},
   "outputs": [],
   "source": [
    "train.to_csv('UNSW_NB15_ohe_freq_enc_training_set.csv',index=False)\n",
    "test.to_csv('UNSW_NB15_ohe_freq_enc_testing_set.csv',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2662df94",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}