EDA.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "a030b435-244e-4e80-84e9-542e44065bd2",
   "metadata": {},
   "source": [
    "# Load basic files, clean duplicates, save"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "b05d2db0-c1d0-4ceb-9f4f-59d6275e2e2b",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import glob\n",
    "import ast\n",
    "import re\n",
    "\n",
    "# All files and directories ending with .txt and that don't begin with a dot:\n",
    "files = glob.glob(\"data_pull/output/*.csv\")\n",
    "files.sort()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "88bbbb6a-fbc5-4be0-81eb-8e29ac6adb44",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>TweetID</th>\n",
       "      <th>AuthorID</th>\n",
       "      <th>AuthorName</th>\n",
       "      <th>Tweets</th>\n",
       "      <th>arxiv_link</th>\n",
       "      <th>Abstract</th>\n",
       "      <th>Title</th>\n",
       "      <th>Thread_length</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>695565872891621376</td>\n",
       "      <td>75249390</td>\n",
       "      <td>Axel Maas</td>\n",
       "      <td>[I have published a new blog entry on our rece...</td>\n",
       "      <td>http://arxiv.org/abs/1601.02006</td>\n",
       "      <td>Observable states are gauge-invariant. In a no...</td>\n",
       "      <td>Gauge invariance and the physical spectrum in ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>694223402932899841</td>\n",
       "      <td>212431150</td>\n",
       "      <td>Faryad Sahneh</td>\n",
       "      <td>[Our new paper: \"Sequential Monte Carlo Filter...</td>\n",
       "      <td>http://arxiv.org/abs/1601.07606</td>\n",
       "      <td>We use a multivariate formulation of sequentia...</td>\n",
       "      <td>Sequential Monte Carlo Filtering Estimation of...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>694093392414937088</td>\n",
       "      <td>2999702157</td>\n",
       "      <td>Anton Ilderton</td>\n",
       "      <td>[New paper on measuring vacuum birefringence u...</td>\n",
       "      <td>http://arxiv.org/abs/1601.08045</td>\n",
       "      <td>The measurement of vacuum polarisation effects...</td>\n",
       "      <td>Prospects for studying vacuum polarisation usi...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>694076172372697088</td>\n",
       "      <td>1430303833</td>\n",
       "      <td>Dave Woods</td>\n",
       "      <td>[New paper on arXiv on screening for GLMs with...</td>\n",
       "      <td>http://arxiv.org/abs/1601.08088</td>\n",
       "      <td>The first investigation is made of designs for...</td>\n",
       "      <td>Model selection via Bayesian information capac...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0             TweetID    AuthorID      AuthorName  \\\n",
       "0           0  695565872891621376    75249390       Axel Maas   \n",
       "1           1  694223402932899841   212431150   Faryad Sahneh   \n",
       "2           2  694093392414937088  2999702157  Anton Ilderton   \n",
       "3           3  694076172372697088  1430303833      Dave Woods   \n",
       "\n",
       "                                              Tweets  \\\n",
       "0  [I have published a new blog entry on our rece...   \n",
       "1  [Our new paper: \"Sequential Monte Carlo Filter...   \n",
       "2  [New paper on measuring vacuum birefringence u...   \n",
       "3  [New paper on arXiv on screening for GLMs with...   \n",
       "\n",
       "                        arxiv_link  \\\n",
       "0  http://arxiv.org/abs/1601.02006   \n",
       "1  http://arxiv.org/abs/1601.07606   \n",
       "2  http://arxiv.org/abs/1601.08045   \n",
       "3  http://arxiv.org/abs/1601.08088   \n",
       "\n",
       "                                            Abstract  \\\n",
       "0  Observable states are gauge-invariant. In a no...   \n",
       "1  We use a multivariate formulation of sequentia...   \n",
       "2  The measurement of vacuum polarisation effects...   \n",
       "3  The first investigation is made of designs for...   \n",
       "\n",
       "                                               Title  Thread_length  \n",
       "0  Gauge invariance and the physical spectrum in ...              1  \n",
       "1  Sequential Monte Carlo Filtering Estimation of...              1  \n",
       "2  Prospects for studying vacuum polarisation usi...              1  \n",
       "3  Model selection via Bayesian information capac...              1  "
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# concatenate all months csv\n",
    "df_all = pd.read_csv(files[0])\n",
    "for i in range(1,len(files)):\n",
    "    df_i = pd.read_csv(files[i])\n",
    "    if len(df_i) > 0:\n",
    "        #df_all.append(pd.read_csv(files[i]))\n",
    "        df_all = pd.concat([df_all, df_i])\n",
    "\n",
    "# evaluate the saved textual-lists to lists of strings \n",
    "df_all['Tweets'] = df_all['Tweets'].apply(lambda x: ast.literal_eval(x))\n",
    "\n",
    "# drop duplicates that arise from overlapping queries. We note that some papers may still have multiple Tweets about them.\n",
    "df = df_all.drop_duplicates(subset=['TweetID'])\n",
    "\n",
    "# add length of tweets\n",
    "df.insert(len(df.columns), \"Thread_length\", df['Tweets'].apply(lambda x: len(x)))\n",
    "df.head(4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "f0740026-7834-42de-bca4-1bff452d188d",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/gz/4p3q1cwx0r9gcrv9d_mdl_vh0000gn/T/ipykernel_57376/888368646.py:21: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df['Tweets'] = df['Tweets_coarse'].apply(lambda x: replace_phrases_clean_concat(x))  # gives a warning; false positive, I think\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>TweetID</th>\n",
       "      <th>AuthorID</th>\n",
       "      <th>AuthorName</th>\n",
       "      <th>Tweets</th>\n",
       "      <th>arxiv_link</th>\n",
       "      <th>Abstract</th>\n",
       "      <th>Title</th>\n",
       "      <th>Thread_length</th>\n",
       "      <th>Tweets_coarse</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>695565872891621376</td>\n",
       "      <td>75249390</td>\n",
       "      <td>Axel Maas</td>\n",
       "      <td>I have published a new blog entry on our recen...</td>\n",
       "      <td>http://arxiv.org/abs/1601.02006</td>\n",
       "      <td>Observable states are gauge-invariant. In a no...</td>\n",
       "      <td>Gauge invariance and the physical spectrum in ...</td>\n",
       "      <td>1</td>\n",
       "      <td>[I have published a new blog entry on our rece...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>694223402932899841</td>\n",
       "      <td>212431150</td>\n",
       "      <td>Faryad Sahneh</td>\n",
       "      <td>Our new paper: \"Sequential Monte Carlo Filteri...</td>\n",
       "      <td>http://arxiv.org/abs/1601.07606</td>\n",
       "      <td>We use a multivariate formulation of sequentia...</td>\n",
       "      <td>Sequential Monte Carlo Filtering Estimation of...</td>\n",
       "      <td>1</td>\n",
       "      <td>[Our new paper: \"Sequential Monte Carlo Filter...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>694093392414937088</td>\n",
       "      <td>2999702157</td>\n",
       "      <td>Anton Ilderton</td>\n",
       "      <td>New paper on measuring vacuum birefringence us...</td>\n",
       "      <td>http://arxiv.org/abs/1601.08045</td>\n",
       "      <td>The measurement of vacuum polarisation effects...</td>\n",
       "      <td>Prospects for studying vacuum polarisation usi...</td>\n",
       "      <td>1</td>\n",
       "      <td>[New paper on measuring vacuum birefringence u...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>694076172372697088</td>\n",
       "      <td>1430303833</td>\n",
       "      <td>Dave Woods</td>\n",
       "      <td>New paper on arXiv on screening for GLMs with ...</td>\n",
       "      <td>http://arxiv.org/abs/1601.08088</td>\n",
       "      <td>The first investigation is made of designs for...</td>\n",
       "      <td>Model selection via Bayesian information capac...</td>\n",
       "      <td>1</td>\n",
       "      <td>[New paper on arXiv on screening for GLMs with...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0             TweetID    AuthorID      AuthorName  \\\n",
       "0           0  695565872891621376    75249390       Axel Maas   \n",
       "1           1  694223402932899841   212431150   Faryad Sahneh   \n",
       "2           2  694093392414937088  2999702157  Anton Ilderton   \n",
       "3           3  694076172372697088  1430303833      Dave Woods   \n",
       "\n",
       "                                              Tweets  \\\n",
       "0  I have published a new blog entry on our recen...   \n",
       "1  Our new paper: \"Sequential Monte Carlo Filteri...   \n",
       "2  New paper on measuring vacuum birefringence us...   \n",
       "3  New paper on arXiv on screening for GLMs with ...   \n",
       "\n",
       "                        arxiv_link  \\\n",
       "0  http://arxiv.org/abs/1601.02006   \n",
       "1  http://arxiv.org/abs/1601.07606   \n",
       "2  http://arxiv.org/abs/1601.08045   \n",
       "3  http://arxiv.org/abs/1601.08088   \n",
       "\n",
       "                                            Abstract  \\\n",
       "0  Observable states are gauge-invariant. In a no...   \n",
       "1  We use a multivariate formulation of sequentia...   \n",
       "2  The measurement of vacuum polarisation effects...   \n",
       "3  The first investigation is made of designs for...   \n",
       "\n",
       "                                               Title  Thread_length  \\\n",
       "0  Gauge invariance and the physical spectrum in ...              1   \n",
       "1  Sequential Monte Carlo Filtering Estimation of...              1   \n",
       "2  Prospects for studying vacuum polarisation usi...              1   \n",
       "3  Model selection via Bayesian information capac...              1   \n",
       "\n",
       "                                       Tweets_coarse  \n",
       "0  [I have published a new blog entry on our rece...  \n",
       "1  [Our new paper: \"Sequential Monte Carlo Filter...  \n",
       "2  [New paper on measuring vacuum birefringence u...  \n",
       "3  [New paper on arXiv on screening for GLMs with...  "
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# in this block we clean media and urls from tweets, replacing by <LINK>, and we concat tweets threads into a single string.\n",
    "\n",
    "def clean_urls(tweet, token='<LINK>'):\n",
    "    urls = re.findall('https?://t.co/[0-9a-zA-Z]+',tweet)\n",
    "    for url in urls:\n",
    "        tweet = tweet.replace(url, token)\n",
    "    return tweet\n",
    "\n",
    "\n",
    "def replace_phrases_clean_concat(tweet_list, replace_dict={'&amp;': '&', '\\n': ' ', '  ': ' '}):\n",
    "    tweet_final = ''\n",
    "    for tweet in tweet_list:\n",
    "        tweet = clean_urls(tweet)\n",
    "        for initial, word in replace_dict.items():\n",
    "            tweet = tweet.replace(initial, word)\n",
    "        tweet_final = tweet_final + ' ' + tweet\n",
    "    tweet_final = tweet_final[1:] # remove extra ' '\n",
    "    return tweet_final\n",
    "\n",
    "df.insert(len(df.columns), \"Tweets_coarse\", df['Tweets'])\n",
    "df['Tweets'] = df['Tweets_coarse'].apply(lambda x: replace_phrases_clean_concat(x))  # gives a warning; false positive, I think\n",
    "\n",
    "df.head(4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "e33cc7ce-ef1b-46b6-844e-26b31d178178",
   "metadata": {},
   "outputs": [],
   "source": [
    "# drop extra column and save full dataset\n",
    "df = df.drop(['Unnamed: 0'], axis=1)\n",
    "df.to_csv('data/full_data.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "81ae30d7-0246-4474-afe2-3edd4123d95d",
   "metadata": {},
   "source": [
    "# General statistical properties"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "f2b77061-0f6e-4d97-837b-fa0cee65dc88",
   "metadata": {},
   "outputs": [],
   "source": [
    "# load full data\n",
    "import pandas as pd\n",
    "df = pd.read_csv('data/full_data.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "b28c4818",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "0\n"
     ]
    }
   ],
   "source": [
    "# compute character length of tweets\n",
    "tweet_lengths = df['Tweets'].apply(lambda x: len(x)).to_numpy()\n",
    "\n",
    "# get year-month from arXiv link\n",
    "import re\n",
    "nums_yr_mon = df['arxiv_link'].apply(lambda x: re.search('\\d+.\\d+',x).group()[0:4])  # in rare cases the arXiv link is weird, so better to regex it\n",
    "\n",
    "years = nums_yr_mon.apply(lambda x: x[0:2])\n",
    "months = nums_yr_mon.apply(lambda x: x[2:])\n",
    "df['year'] = years\n",
    "df['month'] = months\n",
    "\n",
    "# check healthiness of conversion: if any of year/month are not ints, we have a problem\n",
    "def is_not_int(x):\n",
    "    try:\n",
    "        int(x)\n",
    "        return False\n",
    "    except ValueError:\n",
    "        return True\n",
    "# make sure we have ints in year and month. print should yield 0's\n",
    "print(len(df[df['year'].apply(lambda x: is_not_int(x))]))\n",
    "print(len(df[df['month'].apply(lambda x: is_not_int(x))]))\n",
    "\n",
    "# count number of Tweets per year\n",
    "num_per_year = [len(df.loc[df['year']==str(yr_i)]) for yr_i in range(16,23)]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0fc33685-5357-4143-b3ad-0a00bd32882d",
   "metadata": {},
   "source": [
    "### Histogram of tweets with X number of characters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "f6670874-d452-485b-8532-60bfe91db3ca",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEJCAYAAACKWmBmAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAscklEQVR4nO3dd5xU1fnH8c8zbYliDzEKKipGg/qzobELoohKsQt2JSBRbCBNEQELCIKRoCI2FAkoBpUqNpRoLIAlgBWNZZUoatTYdtrz+2OGZLPuLssys3dm9vt+vea1M2fmnvucZdlnzz3nnmPujoiISE1CQQcgIiKFTYlCRERqpUQhIiK1UqIQEZFaKVGIiEitlChERKRWShQiIlIrJQoREalVJOgA6sLMNgQWAVe7+5y1ff6Xv/ylt2zZMu9xiYiUiqVLl37h7s2qey+QRGFmdwOdgM/dfbdK5R2Bm4EwcKe7j8q+NRB4sK71t2zZkiVLluQwYhGR0mZmH9b0XlCXniYDHSsXmFkYuAU4GmgNdDez1mZ2BPAG8FlDBykiIgH1KNx9kZm1rFK8H7DS3d8HMLPpQFegKbAhmeTxo5nNc/d0Q8YrItKYFdIYRXPg40qvy4HfuXsfADM7B/iipiRhZr2AXgDbbrttfiMVEWlECilRWDVl/1na1t0n13awu08ys1VA51gstk+OYxMRabQKaXpsObBNpdctgE/XpQJ3n+3uvTbZZJOcBiYi0pgVUqJYDOxkZtubWQzoBsxalwrMrLOZTfrmm2/yEqCISGMUSKIws2nAC8DOZlZuZj3cPQn0ARYAbwIPuvuKdalXPQoRkdwLatZT9xrK5wHz6luvmXUGOrdq1aq+VYiISBWFNJi93tx9NjC7TZs2PetbR8tBc+v0uQ9GHVvfU4iIFJVCGqNYbxqjEBHJvZJKFBqjEBHJvZJKFCIiknsllSh06UlEJPdKKlHo0pOISO6VVKIQEZHcK6lEoUtPIiK5V1KJQpeeRERyr6QShYiI5J4ShYiI1EqJQkREalVSiUKD2SIiuVdSiUKD2SIiuVdSiUJERHJPiUJERGqlRCEiIrVSohARkVqVVKLQrCcRkdwrqUShWU8iIrlXUolCRERyT4lCRERqpURR2Yu30S70KuBBRyIiUjCUKNZIJeHV+7knNoZ7ozfQysqDjkhEpCAoUawRjkDPhYxInMleoZU8FhvEsMhkNuG7oCMTEQlUwScKM/utmU00s4fM7A95PVkkxt2pozmsYhzTUodzZvgJni27jLPDC4iQzOupRUQKVSCJwszuNrPPzWx5lfKOZva2ma00s0EA7v6mu/cGTgHaNER8/2JjrkqexzHxkSxPt2R49F7mxwZzaOj1hji9iEhBMfeGH7g1s0OB74D73H23bFkYeAc4EigHFgPd3f0NM+sCDAImuPuf11Z/mzZtfMmSJfWKreWguVVKnCNDS7kyMpWWoc94KrUX1yVP533fus51fjDq2HrFIiLSUMxsqbtX+8d4ID0Kd18EfFWleD9gpbu/7+5xYDrQNfv5We5+IHB6TXWaWS8zW2JmS1avXp3DaI0n0m3oEB/N9Ynu7Bd6iwWxgQyJTGFjvs/heUREClMhjVE0Bz6u9LocaG5mbc1svJndDsyr6WB3n+Tubdy9TbNmzXIeXJwok1KdaVcxjhmpQzkv/BgLy/pyevhJwqRyfj4RkUJRSInCqilzd3/G3S929/Pd/ZZaK2iAtZ6+YBOuSPakU/w63vUWXBe9mzmxKzggtCJv5xQRCVIhJYpyYJtKr1sAn65LBQ251tMb3pJu8SH0jl9KU35iWuw6bo+OYzv7Z97PLSLSkAopUSwGdjKz7c0sBnQDZq1LBQ2/eqzxWHo/joiPYXTiVA4OLePx2AAGRf5MU35ooBhERPIrqOmx04AXgJ3NrNzMerh7EugDLADeBB5093W6nhPU6rEVxLg11ZW2FeOYlTqQ3pE5LCzry6nhhYRIN2gsIiK5Fsj02Hwxs85A51atWvV8991361XHz6fHrrvd7X2GRu9j39A7LE+3ZLfzboWWB613vSIi+VLb9NiSShRr5PY+ivpyOodeYFB0Gs3tS+am9mNk8nTKveYZWbrfQkSCUnD3UeRLYe1wZ8xOH0j7ihsZlziJdqHXeSp2OZdHHmADfgo6OBGROiupRFGIO9z9RBnjUydweMWNzEvvR5/Ioyws68uJoUWYxi9EpAiUVKIoZP9kCy5LXMjxFcNZ5VswNjaRh2ND2dveCTo0EZFalVSiKKxLT9V71Xfi+PhwLo1fwK/tX8wsG8bN0QlsxZdBhyYiUi0NZleRu8HstfsFP9E7Mpvzw3NwjNtTnZiY7MxPlNV6nAa9RSTXGs1gdrH5kSbclDyZ9hU38mR6by6NzOTpsn50DT2HtmMVkUJRUomiGC49VecTmnFR4mJOqhjKF74JN8duZWbsavawlUGHJiJSWomiEGc9rYslvgtd49fQP9GLFvYFj5YNZWz0Vrb82YrsIiINp6QSRSlwQsxItaVdxVhuTXahU+hFFpb1o0/4YcqIBx2eiDRCShQF6nt+wehkN46Ij+HZ9P9xeXQGT5VdzrGhF6EEJyCISOEqqVlPhbLWUz7sH3qDoZEptA59yMvpnRmROJPlvkOtx2h2lIjUVaOZ9VTsYxS1eTHdmk7x6xic6MEOtopZsasYHbmdZnwddGgiUuJKKlGUujQhpqXa065iHHekjuG48HMsLOvLH8KzNH4hInmjRFGE/s0GjEyeTof4aF5I78rA6HSeiPXnqNDL6P4LEcm1khqjWKNY7szOlYNCyxgamcLOoXJeSLVmRPJM3vTt6ny8xjJEpNGMUTRWz6d355j4SIYkzmXn0EfMiV3B9ZE72YLiuvFQRApTSSWKYr0zOxdShLk/dSRtK8YxOdWRk8PPsrCsL78PzyVKMujwRKSIlVSiKOVZT3X1LU25JnkmHeOjWJLemSHRqSyIDaB9aCkavxCR+iipRCH/9Z4357zEAM6ODyRNiLtiY7kvOoqdrDzo0ESkyChRlLhn03vQMT6KYYmz2CP0HvNjgxgeuYdN+XfQoYlIkVCiaASSRJic6shhFTcxNdWe08NP8WzZZZwbnk9E4xcishZKFI3I12zE1clzOTo+itfTO3J1dAqPxQbBu08EHZqIFLBI0AFIw3vXW3BWYhDtU69wZWQqTD2Jhak9uDZ5Bu958xqP0/0WIo1TwfcozOw4M7vDzB41sw5Bx1M6jKfS+3BUfDTXJk5nn9A7PBYbxNDIfWzMd0EHJyIFJJBEYWZ3m9nnZra8SnlHM3vbzFaa2SAAd3/E3XsC5wCnBhBuSUsQ4c7UsbSrGMeM1GGcHV7AM2V9OSP8BGFSQYcnIgUgqB7FZKBj5QIzCwO3AEcDrYHuZta60keGZN+XPPiSTbgi+Xs6xa/n7fS2XBu9h7mxKzgotCzo0EQkYIEkCndfBD/b33M/YKW7v+/ucWA60NUybgDmu/srDR1rY/Omb0f3xJWcH7+MX1DB1NhI7oiOpaWtCjo0EQlIIY1RNAc+rvS6PFt2EXAEcJKZ9a7pYDPrZWZLzGzJ6tWr8xtpyTMWpPelQ3w0oxLdOCC0gsdjA+DxIfBT41seRaSxK6REYdWUubuPd/d93L23u0+s6WB3nwQMB16JxWJ5C7IxqSDGxFQX2lWM4+HUIfC3CTB+b1g6GdIavxBpLAppemw5sE2l1y2AT9elAnefDcxu06ZNz1wG1titZlMGJnsxJXUEQ9NT2G/2Jbzx6DhGJM/kxXTrn31e02hFSksh9SgWAzuZ2fZmFgO6AbPWpYLGvHpsQ1juO3BKfCgXxi9mY/ue6bFruS16E9vYZ0GHJiJ5FNT02GnAC8DOZlZuZj3cPQn0ARYAbwIPuvuKdalXq8c2BGNuen/aV9zIjYmTOSz0d56M9WdAZDob8mPQwYlIHgRy6cndu9dQPg+YV996zawz0LlVq1b1rULqqIIYE1LHMyN1GAOi07kgMouTwosYkzwF0kdDqJA6qyKyPrQVahXFuBVqIdjTVjI0eh97h1ayLN2S4YmzWOK71HqMxjJECoe2QpW8e81bcUJ8OBfHL2QL+5aHykYwITqe5miqskixK6lEocHsoBmz0gfRvuJG/pg8gfahV3iq7HL6Rh7kF/wUdHAiUk8llSg0mF0YfqQJf0yexOEVY1mQ3peLI4+wsKwfx4f+ipEOOjwRWUcllSjUoygsq9iCSxJ9OKFiGJ/5ZtwUu42ZsWHsZe8GHZqIrIOSShTqURSmV/w3HBcfQb94b7a2L3i47Gpuit4C33wSdGgiUgcllSikcDkh/pI+lHYV45iQ7MoxoZdhQht45gaI/xB0eCJSCyUKaVA/0IQbk6fSPj4GduoAz1wPE/aFZQ9BCU7VFikFhbTW03rTDXfFo9x/RctXTmE/252hX09ht7/0YPGMGxiROItlvsPPPq97LkSCU1I9Co1RFJ+X/bd0iV/LgERPWto/mV02hDGRiTTjX0GHJiJZJZUopDilCfFgqh3tKsYxMdmZLuG/8UxZXy4IP0IZ8aDDE2n0lCikYHzHBoxKdufI+BieS+/OgOiDPBnrz9GhlzR+IRKgkkoUuo+iNHzkW3J+oi/d41fyHU24LXYzTO4Eq/4edGgijVJJJQqNUZSWF9K70il+PVcmzoPVb8Lth8Ksi+C7z4MOTaRRWeusJzPbEPjR3dNm9htgF2C+uyfyHp00einCTE0dweyvDuDiyEzOXjqVn5bO4E/J45ic6kic6P98XrOjRHKvLj2KRUATM2sOPAWcC0zOZ1AiVX3LhlybPJOj4jfwcnoXrohO4/HYAI4MLQE0fiGST3VJFObuPwAnAH9y9+OBn2+ULNIA3vet6ZHoz1nxgSSIcEdsHPdHr2dn+yjo0ERKVp0ShZkdAJwOrNnVp6Ru1JPisyi9B0fHRzI0cTa7hT5gXmww10Tuhu+/CDo0kZJTl0RxCTAYeNjdV5jZDsDC/IZVP5r11LgkiXBf6igOq7iJ+1Id6B5+GsbvDS/cAkndfyGSK3VJFFu6exd3vwHA3d8H/prfsOpHs54ap29oyvDk2XSMj4IWbWDBFXDbAfDOAt1/IZIDdUkUg+tYJhKold4CzvgLnPZgpuDPp8D9J8LnbwUbmEiRq3GswcyOBo4BmpvZ+EpvbQwk8x2YSL2YwW+Ogh3aweI7MsuY33Yg7Pt7aDsINtgcgJaD5q6logxNtxWpfVD6U2AJ0AVYWqn838Bl+QxKpL7+NwG0ZHNu4LLIQ5z20iS+fWkqNyVPZGrqCCAcVIgiRafGS0/u/rq73wu0Ah4EXnT3e919prtraU8pCl+xMVclz+OY+EhWpLdjRPRe5scGcUhIy4GI1FVdxig6Aq8BjwGY2Z5mNiufQYnk2tu+LWckrqBnvC8xkkyJjeLO6Bi2t1VBhyZS8OqSKIYB+wFfA7j7a0DLfAVUlZntYGZ3mdlDDXVOKVXGE+k2dIiP5vpEd34XeovHYwMYEpnCxnwfdHAiBasuiSLp7jm9McHM7jazz81seZXyjmb2tpmtNLNBkJmO6+49cnl+adziRJmU6ky7inE8lDqU88KPsbCsL6eHnyREOujwRApOXRLFcjM7DQib2U5m9ifgb+t53slkLmn9h5mFgVuAo8ksEdLdzLRUiOTNF2zC4GRPOsevY6U357ro3cyNDeaA0IqgQxMpKHVJFBcBuwIVwJ+Bb4BL1+ek7r4I+KpK8X7AymwPIg5MB7quz3lE6mKFt+TU+FX8IX4JTfmJabHruD06jm3ts6BDEykIa00U7v6Du18JtHX3fd19iLv/lIdYmgMfV3pdTuYeji3MbCKwl5nVeKOfmfUysyVmtmT16tV5CE9KmzE//TuOiI9hdOIUDg4t44lYf3hiKPz0bdDBiQSqLvtRHAjcCTQFtjWzPYDz3f2CHMdi1ZS5u38J9F7bwe4+ycxWAZ1jsdg+OY5NGokKYtyaOo6HUocxIPoAJz1/M6ufm8yY5Ck8lDqM9Hru9aUb+KQY1eWn/ibgKOBLyNxfARyah1jKgW0qvW5B5qa/OtNaT5Irn7MZlyd607niWj70LRkdvYNZsSHsa1oORBqfOv155O4fVylK5SGWxcBOZra9mcWAbsA63a+h1WMl15b5DpwUv5qL4n3YzP7NjLIRTIjeTAvT5U1pPOqSKD7OXn5yM4uZ2eXAm+tzUjObBrwA7Gxm5WbWw92TQB9gQbb+B919naafqEch+WHMTh9I+4obGZc4ifahV3kqdjn9Ig+yAfkYrhMpLHXZgKg3cDOZweZy4HHgwvU5qbt3r6F8HjCvvvWaWWegc6tWrepbhUiNfqKM8akTeDB1GAOj07ko8ginhJ9hdKIbM9MH4+s5flGZFi2UQlKXn+zv3P10d9/S3X/l7mdkB5gLjnoU0hD+yRZclriQ4yuGs8q3YGxsIg/HhrK3vRN0aCJ5UZcexXIz+4zMZkWLgOdzfad2rqhHIQ3pVd+J4+PDOS70PAOj05lZNoxHUwcyKtGdVWxR7TF17SmIFJK63EfRCugOLAM6Aa+b2Wt5jqte1KOQhuaEeDh9CIdXjGV88jiOCi3m6bJ+XBp5iCZUBB2eSE6sNVGYWQvgIOAQYC9gBfBAnuMSKSo/0IRxyVNoX3EjT6X35tLITJ4u60eX0POAtmOV4laXMYqPyCzZMd/dD3D3Y919ZH7Dqh9Nj5WgfUIz+iQu5uSKoXzpGzM+dgszY1ezh60MOjSReqsxUZjZmvGLvYD7gNPM7AUzu8/MCnI1V116kkKx2HehS/xa+id6sY2t5tGyoYyN3sqWP1viTKTw1dajeBn+cyf2vcA9wNPAYcBV+Q9NpLg5IWak2tK2Yhy3JrvQKfQiC8v60Sf8MGXEgw5PpM7qMkaxhMzNcceTuRHuUHdvmee46kWXnqQQfc8vGJ3sxhHxMTyb/j8uj87gqbLLOTb0Ihq/kGJQW6L4lZn1JTNwPRl4B/gVcGK2vODo0pMUso99S/6QuIzu8Sv51jfklth4Hohdw672j6BDE6lVbYkiTGbF2A2yX5sCG1V6iEg9vJDelU7x6xic6MGO9imzY0O4ITKJZpndhkUKTm033K1y9xENFolII5ImxLRUe+akDqBP5GHODT/GMeGXuCXZlbtTRxMnGnSIIv9RW4+iuv0hRCSH/s0GjEyeTof4aF5Mt2ZQdDpPxPpzVGgxGr+QQlFbj6J9g0WRI1rCQ4rVB74VPRP9ODi1jKsiU7g9dhN/S7VmRPIs3vJtGzSWoBYk1EKIhavGHoW7F92Ebw1mS7F7Lr07x8RHMiRxLruEPmJubDDXR+5kc7QdqwQnd+sii0hOpAhzf+pI2laM497UUZwcfpZnyi7j9+G5REkGHZ40QkoUIgXqW5oyInkWHeOjWJr+DUOiU1kQG0D70FI0fiENSYlCpMC95805NzGQc+IDSBPirthY7ouOYicrDzo0aSSUKESKxDPpPekYH8XwxJnsEXqP+bFBMPdy+KHohhOlyNRl46KioVlPUuqSRLgndTSPpA7isshfOGvJXbBsBrQdDPv2gHDN9180tk2TNIsqd0qqR6FZT9JY/IuNGZo8F3o/D1vvCY8NhNsOgnefDDo0KUEllShEGp0tW8OZj0D36ZBOwNQTYerJ8MW7QUcmJUSJQqTYmcHOR8MFL0GHa+GjF+HW/eGxwfDjv4KOTkqAEoVIqYjE4MCL4KJXYK8z4MXbYPzesPhOSOn+C6k/JQqRUtO0GXS+GXr/FbbcFeb2g9sP4aDQsqAjkyJV8InCzDY0s3vN7A4zOz3oeESKxq93h7NnwylTIP49U2MjuSM6lpa2KujIpMgEkijM7G4z+9zMllcp72hmb5vZSjMblC0+AXjI3XsCXRo8WJFiZgatu8CFL3NDohsHhFbweGwAgyNT2Ygfgo5OikRQPYrJQMfKBWYWBm4BjgZaA93NrDXQAvg4+7FUA8YoUjqiTbgt1YV2FWN5JHUwPcPzWFjWl27hpwmRDjo6KXCBJAp3XwRUvZ10P2Clu7/v7nFgOtAVKCeTLKAILpWJFLLVbMaA5Pl0iV/DP/zXjIreyZzYlewfeiPo0KSAmXswi4uZWUtgjrvvln19EtDR3X+ffX0m8DtgIDAB+Al4zt2n1lBfL6AXwLbbbrvPhx9+WK+4Gtvdq9KYOZ1CLzIoOo0W9gXzU/tyffI0PvYtgw6sYNX1Lu5c3xXeEHeZm9lSd29T3XuFtIRHdTvqubt/D5y7toPdfRIwCaBNmzZaWlNkrYw56QN4omIfeobnckFkFofHXuWu1DHckuzK9/wi6AClQBTSpZxyYJtKr1sAn65LBWbW2cwmffPNNzkNTKSUVRBjQup42lWMZU56fy6IzGJhWT9ODj+DafxCKKxEsRjYycy2N7MY0A2YtS4VaK0nkfr7jM3pl7iA4ypGUO6/ZEx0Eo/GrqKNvRV0aBKwoKbHTgNeAHY2s3Iz6+HuSaAPsAB4E3jQ3VesY73qUYisp9e8FSfEh3NJ/AKa2Tc8VDaCP0XHszVfBB2aBCSQMQp3715D+Txg3nrUOxuY3aZNm571rUNEAIxH0wfzeEUbekfmcH54NkeWLeX2VCcmJjvzI02CDlAaUCFdelpv6lGI5NaPNOGm5EkcXjGWBel9uSTyME+XXc5xoec0ftGIlFSi0BiFSH58yi+5JNGHEyqG8blvyh9jtzIzNow9bWXQoUkDKKlEoR6FSH694r/huPgI+sV7s7V9wSNlQxkXvZUtf3b/rJSSkkoU6lGI5J8T4i/pQ2lXMY4Jya4cG3qJhWX9uDg8kyZUBB2e5EFJJQoRaTg/0IQbk6fSPj6Ghek96Bt9iKfKLqdz6G+A7nktJSWVKHTpSaThlfuvuDBxKadWXMXX3pQ/xSYwIzac3e39oEOTHCmpRKFLTyLBecl/S+f4dQxM9KSl/ZNHY1cxJjKRZmg71mJXUolCRIKVJsQDqXa0qxjHpNSxdA0/z8KyflwQfpQy4kGHJ/VUUolCl55ECsN3bMCo5GkcGR/D39K7MiD6AE/E+tMx9DIavyg+JZUodOlJpLB86L+mV6Ifp8Wv4AeaMDH2R6bHrqW1fRB0aLIOSipRiEhh+lt6N46NX8+VifP4jX3MnNiVXB+5gy1Q778YKFGISINIEWZq6gjaVtzE3amOnBxexMKyvvQMzyFGIujwpBYllSg0RiFS+L5lQ65NnslR8RtYnN6FK6N/5vHYAI4MLUHjF4WppBKFxihEisf7vjU9Ev05Kz6QBBHuiI3j/uj17GwfBR2aVFFSiUJEis+i9B4cHR/J1Ymz2S30AfNig7kmcjeb8W3QoUmWEoWIBC5JhHtTR9G2YhxTUkfSPfw0z5T15bzwfCIkgw6v0VOiEJGC8TUbMSx5DkfHR/F6ekeGRqewIDaQtqFXgw6tUVOiEJGC86634KzEIM6N9wdgcmwMk6M3sKN9EnBkjVNJJQrNehIpJcbC9F50jN/ANYkz2Dv0LgtiA7k6ci+b8F3QwTUqJZUoNOtJpPQkiHBX6hjaVoxjeqodZ4Uf55myvpwVXkCYVNDhNQollShEpHR9xcYMSfbg2PhI3khvx4jovcyPDeKQ0N+DDq3kKVGISFF5y7fl9MQV9IpfRowkU2KjuDM6hu1tVdChlSwlChEpQsbj6X3pEB/NyER3fhd6iwWxAVwZuZ+N+T7o4EqOEoWIFK04UW5PdaZdxThmpg6hR3g+C8v6clr4KUKkgw6vZChRiEjR+4JNGJTsRef4daz05lwfvYu5sSs4ILQi6NBKQsEnCjPbwczuMrOHgo5FRArbCm/JqfGr+EP8EpryI9Ni1zExehPb2GdBh1bU8poozOxuM/vczJZXKe9oZm+b2UozG1RbHe7+vrv3yGecIlJKjPnp33FEfAyjE6dwSOjvPBnrz8DINJryQ9DBFaV89ygmAx0rF5hZGLgFOBpoDXQ3s9ZmtruZzany+FWe4xORElVBjFtTx9GuYhyz0wfyh8hsFpb145TwQo1frKO8Jgp3XwR8VaV4P2BltqcQB6YDXd19mbt3qvL4PJ/xiUjp+5zNuDzRmy4V1/Chb8no6B3Mig1hX3sr6NCKRhBjFM2Bjyu9Ls+WVcvMtjCzicBeZja4ls/1MrMlZrZk9erVuYtWRErC331HTopfzcXxPmxu3zKjbAQTojfTwvT7Ym0iAZzTqimrcVsrd/8S6L22St19kpmtAjrHYrF91iM+ESlZxqz0gTxesQ/nh+fQOzKbI2OvMCl1LLclu/ADTYIOsCAF0aMoB7ap9LoF8GkuKtZaTyJSFz9Rxs2pEzm8Yizz0/tyUeQRni7rxwmhRZjGL34miESxGNjJzLY3sxjQDZiVi4q1eqyIrItVbMGliT6cUDGMf/pmjItN5OHY1ext7wQdWkHJ9/TYacALwM5mVm5mPdw9CfQBFgBvAg+6e07uilGPQkTq4xX/DcfHR9A33put7Etmlg3j5ugEtuLLoEMrCHkdo3D37jWUzwPm5fp8ZtYZ6NyqVatcVy0iJc4JMTN9KI9V7EfvyCx6hefSoWwJt6c6MTHZmZ8oCzrEwBT8ndnrQj0KEVlfP9CEcclTaF9xI0+l9+bSyEyeLutHl9Dz4DXOuylpJZUoNEYhIrnyCc3ok7iYkyuG8qVvzPjYLXBXB/hkadChNbiSShTqUYhIri32Xegav5b+iV7wrw/gjsPh4d7wbePZ/6KkEoWISD6kCTEj1RYufgUOvgyW/wX+tA8sGgOJH4MOL+9KKlHo0pOI5FXZRnDEMLjwZWh1ODx9LUzYD1Y8XNLjFyWVKHTpSUQaxObbw6n3w9mzocnGMOMcuOcYWPV60JHlRUklChGRBrX9oXD+Iuj0R/jiHbj9MHj0QprxddCR5ZQShYjI+giFoc25mfGLAy6E1x/g6bJ+9A7PIkYi6OhyoqQShcYoRCQwTTaBo66DC1/ixfRvGRSdzhOx/hwVWkwt654WhZJKFBqjEJHAbbEjPROXc0Z8MD8R4/bYTfw5eh272EdBR1ZvJZUoREQKxXPp3TkmPpKrEufw29BHzI0N5rrIXWzOt0GHts6UKERE8iRFmCmpDrStGMd9qQ6cGl7IM2V96RGeS5Rk0OHVWUklCo1RiEgh+oamDE+ezVHxG3glvRNXRafyWGwgvP1YUdx/UVKJQmMUIlLI3vPmnJMYyDnxAZmCaafC/SfA54W9f3dJJQoRkWLwTHpPjorfAB1HZRYZvO1AmNcffvgq6NCqpUQhIhKAJBHY/w9w0auZ+zAW3wnj94KXbodUYd1/oUQhIhKkDbeAY8dC7+dh6z1h/gC47SB498mgI/sPJQoRkUKwZWs48xHoNg3SCZh6Ikw9Gb54N+jISitRaNaTiBQ1M9jlGLjgRTjyGvjoRbh1f66KTGFjvgssrJJKFJr1JCIlIVIGB10MF70Ce57OueHHeKasL2eEnyBMqsHDKalEISJSUpo2gy7j6RS/jnd8G66N3sPc2BUcFFrWoGEoUYiIFLg3vCXd4kPoHb+UDfmJqbGR3BEdy3b2zwY5vxKFiEhRMB5L78cR8TGMTpzKAaEVPBHrz+DIVDbih7yeWYlCRKSIVBDj1lRX2lWM5ZHUwfQMz+Ppsr50Cz8N6fyMXyhRiIgUodVsxoDk+XSJX8MH/mtGRe+ESYfBv3N/OargE4WZHWdmd5jZo2bWIeh4REQKyXLfgZPjV9MnfhFsuh1s+KucnyOvicLM7jazz81seZXyjmb2tpmtNLNBtdXh7o+4e0/gHODUPIYrIlKkjDnpA6DbVAjl/td6JOc1/q/JwATgvjUFZhYGbgGOBMqBxWY2CwgDI6scf567f559PiR7nIiINKC8Jgp3X2RmLasU7wesdPf3AcxsOtDV3UcCnarWYWYGjALmu/srNZ3LzHoBvQC23Xbb3DRAREQCGaNoDnxc6XV5tqwmFwFHACeZWe+aPuTuk9y9jbu3adasWW4iFRGRvF96qo5VU1bjFk/uPh4YX6eKzToDnVu1alXP0EREpKogehTlwDaVXrcAPs1FxVrrSUQk94JIFIuBncxsezOLAd2AWbmoWKvHiojkXr6nx04DXgB2NrNyM+vh7kmgD7AAeBN40N1X5OJ86lGIiORevmc9da+hfB4wL9fn0xiFiEjumXuN48hFy8xWAx/W8/BfAl/kMJygqB2FRe0oHKXQBsh9O7Zz92qnjJZkolgfZrbE3dsEHcf6UjsKi9pROEqhDdCw7Sj4tZ5ERCRYShQiIlIrJYqfmxR0ADmidhQWtaNwlEIboAHboTEKERGplXoUIiJSKyWKrHXZIyMIZraNmS00szfNbIWZXZIt39zMnjCzd7NfN6t0zOBse942s6Mqle9jZsuy743PrtDbkG0Jm9mrZjanWNuQjWFTM3vIzN7K/rscUGxtMbPLsj9Py81smpk1KZY2VLffTS5jN7MyM3sgW/6S/Xwl7Hy1YUz2Z+rvZvawmW0aeBvcvdE/yOyF8R6wAxADXgdaBx1XlRi3AvbOPt8IeAdoDYwGBmXLBwE3ZJ+3zrajDNg+275w9r2XgQPILNA4Hzi6gdvSF/gzMCf7uujakI3hXuD32ecxYNNiaguZVZv/Afwi+/pBMhuEFUUbgEOBvYHllcpyFjtwATAx+7wb8EADtaEDEMk+v6EQ2tCg/7EK9ZH9Bi+o9HowMDjouNYS86NkNn96G9gqW7YV8HZ1bSCzZMoB2c+8Vam8O3B7A8bdAngKOJz/JoqiakP2nBuT+SVrVcqLpi38d8n/zcms0jAn+0uqmNrQssov2ZzFvuYz2ecRMje3Wb7bUOW944GpQbdBl54y1nWPjEBlu497AS8BW7r7KoDs1zUb5tbUpubZ51XLG8ofgQFAulJZsbUBMr3P1cA92ctod5rZhhRRW9z9E+BG4CNgFfCNuz9OEbWhGrmM/T/HeGaNum+ALfIWefXOI9ND+J94shqsDUoUGeu0R0aQzKwp8BfgUnf/traPVlPmtZTnnZl1Aj5396V1PaSaskDbUEmEzCWD29x9L+B7Mpc6alJwbclev+9K5jLG1sCGZnZGbYdUU1Yo/x5rU5/YA22XmV0JJIGpa4kn721QosjI2x4ZuWRmUTJJYqq7z8wWf2ZmW2Xf3wpYs8d4TW0qzz6vWt4QDgK6mNkHwHTgcDO7n+JqwxrlQLm7v5R9/RCZxFFMbTkC+Ie7r3b3BDATOJDiakNVuYz9P8eYWQTYBPgqb5FXYmZnk9ka+nTPXjciwDYoUWTkbY+MXMnOYrgLeNPdx1V6axZwdvb52WTGLtaUd8vOetge2Al4Odsd/7eZ7Z+t86xKx+SVuw929xbu3pLM9/hpdz+jmNpQqS3/BD42s52zRe2BNyiutnwE7G9mG2TP3Z7M0v/F1Iaqchl75bpOIvPzmvcehZl1BAYCXdz9h0pvBdeGhhhwKoYHcAyZmUTvAVcGHU818R1Mpsv4d+C17OMYMtcbnwLezX7dvNIxV2bb8zaVZqEAbYDl2fcmkIcBujq0py3/Hcwu1jbsCSzJ/ps8AmxWbG0BhgNvZc8/hcyMmqJoAzCNzNhKgsxfzj1yGTvQBJgBrCQzq2iHBmrDSjLjCmv+n08Mug26M1tERGqlS08iIlIrJQoREamVEoWIiNRKiUJERGqlRCEiIrVSopBAmJmb2dhKry83s2EBxjPGMquojqnj57/Ld0y5ZGZ7mtkxeax/UzO7IF/1S7CUKCQoFcAJZvbLoAPJOp/M6rz9832i7B2yDX2OPcncd5Mvm5JZqVRKkBKFBCVJZivHy6q+YWaTzeykSq+/y35ta2bPmtmDZvaOmY0ys9PN7OXsWvw71nZCyxhjmb0XlpnZqdnyWcCGwEtryiod09TM7sl+/u9mdmKl964zs9fN7EUz2zJb1jm77v+rZvZkpfJhZjbJzB4H7jOzXbNxv5atd6dq4v3OzMaa2Stm9pSZNcuW72hmj5nZUjP7q5ntUun7Ns7MFpJZnnpNPTFgBHBq9nynZtuzafZ78qWZnZX97BQzO8Iye4aMMbPF2fjOr1Rf/0rlw7PFo4Ads/WPMbOtzGxR9vVyMzuktn8bKXANdQeoHnpUfgDfkVmq+wMy689cDgzLvjcZOKnyZ7Nf2wJfk1lWuQz4BBiefe8S4I9rOeeJwBNk9h/ZkswSFltVPkc1x9xQuV5gs+xXBzpnn48Ghqx5n//eFft7YGz2+TBgKf/d++FPZNbxgcxeFr+o5txe6TNDgQnZ508BO2Wf/47Msgxrvm9zyO5RUKWuc9Ycn309ETgW2I3MEjZ3ZMvfBZoCvSq1qYzMHejbk1mGfBKZxeZC2fMdys+X++5HdoWD7Pd7o6B/5vSo/yPvXWCRmrj7t2Z2H3Ax8GMdD1vs2WWkzew94PFs+TKg3VqOPRiY5u4pMovHPQvsS+3reh1BZl2qNTH/K/s0TuaXJGQSwJHZ5y2AByyzIF2MzJ4Va8xy9zXtfAG40sxaADPd/d1qzp0GHsg+vx+YaZnVgw8EZth/N5Erq3TMjGz71uavZH7BfwjcBvQys+bAV+7+nZl1AP6vUs9uEzJrC3XIPl7NljfNln9Upf7FwN2WWcjyEXd/rQ4xSYHSpScJ2h/JrG+zYaWyJNmfzewiZ7FK71VUep6u9DoNa/3Dpz7bcxrVL8uc8Oyfy0Cq0rn/ROYv993JjHs0qXTM92ueuPufgS5kEuQCMzu8DrE4me/L1+6+Z6XHb6s7x1osAg7JPp4hs7fGSWQSCGTafVGlc2zvmb0qDBhZqbyVu9/1s0DdF5FJRJ8AU9Zc2pLipEQhgXL3r8hswdmjUvEHwD7Z512BaI5Ot4jMdfpw9nr/oWQWSqvN40CfNS+s0h7MNdiEzC9H+O+qnT9jZjsA77v7eDI9mv+r5mMhMr+8AU4DnvPMHiT/MLOTs/WYme2xlpgA/k1mC10A3P1j4JdkLmG9DzxH5vLfmkSxAPhDtkeAmf3GMhszLQDOy/ZsMLPmZvarqvWb2XZk9h65g8yqx3vXIUYpUEoUUgjGkvmltcYdwGFm9jKZa/B1/SsZADPrYmYjqnnrYTIrvb4OPA0M8Mxy4bW5FtgsOyD7Omu/vDWMzGWhv5LZdrImpwLLzew1YBfgvmo+8z2wq5ktJbN17Jo2nQ70yMazgkwyXZuFQOs1g9nZspfIrJgMmQTRnEzCALiTzLLpr5jZcuB2Mvs4P05mv/MXzGwZmX04NnL3L4Hns9+nMWTGk14zs1fJjA3dXIcYpUBp9ViRAmVm37l706DjEFGPQkREaqUehYiI1Eo9ChERqZUShYiI1EqJQkREaqVEISIitVKiEBGRWilRiIhIrf4fUJbGQt0qVpIAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "\n",
    "plt.hist(tweet_lengths, bins=30)\n",
    "plt.xlabel('Num. of chars per tweets')\n",
    "plt.ylabel('Tweets')\n",
    "plt.yscale(\"log\")\n",
    "\n",
    "c1 = np.arange(0.0, 12200, 1)\n",
    "t1 = 1400*np.exp(-c1/1050)\n",
    "\n",
    "plt.plot(c1,t1)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ce18fb11-3c24-4f21-bdef-003b64e8c3c2",
   "metadata": {},
   "source": [
    "We find a middle regime that follows an exponential law, and somewhat different behavior for small and large number of characters."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "44a75129-7c85-4678-8c7b-d737e1a3d31d",
   "metadata": {},
   "source": [
    "### Evolution of tweets on papers per year"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "873dc3b4-f138-424d-81ab-00f2ffe19024",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEGCAYAAACKB4k+AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAdY0lEQVR4nO3de5RcZZnv8e/PgGLTqCDSZoB0ixOYATxGu4fxjBfSYRDwAooDktN6orhsddCDl3EJBgXHyRqvcM7xSisZo7Q0eABBZFSEdJClERNOhISLBOjESE7iBUw60WCS5/yxdxWVpqt6d1VX7erq32etWrX3u2/PQy36yd773ftVRGBmZgbwtLwDMDOz5uGiYGZmRS4KZmZW5KJgZmZFLgpmZla0X94B1OLQQw+Nrq6uqrffsWMHBx544NQFlJNWyQOcSzNqlTzAuRSsXr36dxHxvPGWTeui0NXVxapVq6refnh4mPnz509dQDlplTzAuTSjVskDnEuBpA3llvnykZmZFbkomJlZkYuCmZkVuSiYmVmRi4KZmRW5KJiZWZGLgpmZFbkomJlZkYuCmbW8wUHo6oIFC06kqyuZt/FN6yeazcwmMjgI/f2wcyeA2LAhmQfo68szsubkMwUza2mLFxcKwpN27kza7alcFMyspW3cOLn2mc5Fwcxa2pw5k2uf6VwUzKylLVkCbW37trW1Je32VC4KZjauVumx09cHAwPQ2QlS0NmZzE/Xm8z1/l3qVhQkHSlpuaT7JK2TdH7afoikWyQ9mH4fXLLNhZLWS3pA0in1is3MKiv02NmwASKe7LEznQvDyAjcdtsKRkamd0Go9+9SzzOF3cCHIuJvgZcB50k6FrgAuDUi5gK3pvOky84BjgNOBb4saVYd4zOzMtxjpzk14nepW1GIiM0RcVc6vR24DzgcOANYlq62DHhDOn0GMBQRuyLiEWA9cEK94jOz8txjpzk14ndpyMNrkrqAlwA/BzoiYjMkhUPSYelqhwMrSzbblLaN3Vc/0A/Q0dHB8PBw1XGNjo7WtH2zaJU8wLk0i8MOexlbthwwTvufGR5eOc4W08N0/k2gQb9LRNT1A7QDq4Ez0/nHxyx/LP3+EvCWkvYrgDdV2nd3d3fUYvny5TVt3yxaJY8I59Isrrwyoq0tAp78tLUl7dPZdP5NIqbudwFWRZm/q3XtfSRpf+BaYDAirkubt0ianS6fDWxN2zcBR5ZsfgTwaD3jM7PxtVqPnVbRiN+lnr2PRPKv/fsi4tKSRTcCi9LpRcANJe3nSHqGpBcAc4E76xWfmVXWKj12Wk29f5d63lN4OfBW4B5Ja9K2jwKfAq6R9A5gI3AWQESsk3QNcC9Jz6XzImJPHeMzM7Mx6lYUIuIOQGUWn1RmmyWAnzM0M8uJn2g2M7MiFwUzMytyUTAzsyIXBTMzK3JRMDOzIhcFMzMrclEwM7MiFwUzMytyUTAzsyIXBTMzK3JRMDOzIhcFMzMrclEwM7MiFwUzMytyUTAzsyIXBTMzK6rncJxLJW2VtLak7WpJa9LPSGFENkldkv5Usuyr9YrLzMzKq+dwnN8Avgh8s9AQEW8uTEv6PPDHkvUfioh5dYzHzMwmUM/hOG+X1DXeMkkCzgYW1Ov4ZmY2eYqIiVeS/gHooqSIRMQ3y27w5HZdwE0RcfyY9lcBl0ZET8l664BfAduAiyLiJ2X22Q/0A3R0dHQPDQ1NGH85o6OjtLe3V719s2iVPMC5NKNWyQOcS0Fvb+/qwt/fsSY8U5D0LeCFwBpgT9oclFwWqsJC4KqS+c3AnIj4vaRu4LuSjouIbWM3jIgBYACgp6cn5s+fX3UQw8PD1LJ9s2iVPMC5NKNWyQOcSxZZLh/1AMdGllOKDCTtB5wJdBfaImIXsCudXi3pIeBoYNVUHNPMzLLJ0vtoLfD8KTzmPwL3R8SmQoOk50malU4fBcwFHp7CY5qZWQZlzxQkfY/kMtFBwL2S7iT91zxARJxeaceSrgLmA4dK2gRcHBFXAOew76UjgFcB/yppN8klqndHxB8mn46ZmdWi0uWjz9Wy44hYWKb9beO0XQtcW8vxzMysdmWLQkSsAJD06Yj4SOkySZ8GVtQ5NjMza7As9xROHqfttKkOxMzM8lfpnsJ7gH8GjpJ0d8mig4Cf1jswMzNrvEr3FL4N/Cfw78AFJe3bfRPYzKw1Vbqn8EeSdxMtTLuLdqTrt0tqj4iNDYrRzMwaJMsTze8FLgG2AHvT5gD+S/3CMjOzPGR5ovn9wDER8fs6x2JmZjnL0vvo1+z7imszM2tRWc4UHgaGJX2ffZ9ovrRuUZmZWS6yFIWN6efp6cfMzFrUhEUhIj4BIOmgZDZG6x6VmZnlYsJ7CpKOl/R/Sd6Wuk7SaknH1T80MzNrtCw3mgeAD0ZEZ0R0Ah8CvlbfsMzMLA9ZisKBEbG8MBMRw8CBdYvIzMxyk6n3kaSPAd9K598CPFK/kMzMLC9ZzhTOBZ4HXAdcn06/faKNJC2VtFXS2pK2SyT9RtKa9POakmUXSlov6QFJp0w+FTMzq1WW3kePAf+jin1/A/gi8M0x7ZdFxD4D+Eg6lmREtuOAvwJ+LOnoiNhTxXHNzKxKWd591AN8FOgqXT8iKr77KCJul9SVMY4zgKGI2AU8Imk9cALws4zbm5nZFMhyT2EQ+DBwD0++EK8W75X034FVwIfSM5HDgZUl62xK28zMrIEUEZVXkO6IiFdUtfPkTOGmiDg+ne8AfkfyltVPArMj4lxJXwJ+FhFXputdAdycjt08dp/9QD9AR0dH99DQUDWhATA6Okp7e3vV2zeLVskDnEszapU8wLkU9Pb2ro6InvGWZTlTuFjS14Fb2ffdR9dNNpCI2FKYlvQ14KZ0dhNwZMmqRwCPltnHAMmzE/T09MT8+fMnG0bR8PAwtWzfLFolD3AuzahV8gDnkkWWovB24G+A/dl3PIVJFwVJsyNiczr7RpKnpAFuBL4t6VKSG81zgTsnu38zM6tNlqLw4oh40WR3LOkqYD5wqKRNwMXAfEnzSIrKCPAugIhYJ+ka4F5gN3Ceex6ZmTVelqKwUtKxEXHvZHYcEQvHab6iwvpLgCWTOYaZmU2tLEXhFcAiSY+Q3FMQydtSPRynmVmLyVIUTq17FGZm1hSyPNG8oRGBmJlZ/rK8+8jMzGYIFwUzMyuqWBQkzZL040YFY2Zm+apYFNJnBXZKenaD4jGb9gYHoasLFiw4ka6uZN5susjS++jPwD2SbgF2FBojoprXaZu1tMFB6O+HnTsBxIYNyTxAX1+ekZllk6UofD/9mNkEFi8uFIQn7dyZtLso2HSQpUvqMknPBOZExAMNiMls2tq4cXLtZs1mwt5Hkl4PrAF+kM7Pk3RjneMym5bmzJlcu1mzydIl9RKSUdAeB4iINcAL6haR2TS2ZAm0te3b1taWtJtNB1mKwu6I+OOYtsoj85jNUH19MDAAnZ0gBZ2dybzvJ9h0kaUorJX034BZkuZK+gLw0zrHZTZt9fXByAjcdtsKRkZcEGx6yVIU3gccR/KG1KuAbcD76xiTmZnlJEvvo53AYkmfTmZje/3DMjOzPGTpffR3ku4B7iZ5iO2XkrozbLdU0lZJa0vaPivpfkl3S7pe0nPS9i5Jf5K0Jv18tYaczMysSlkuH10B/HNEdEVEF3Ae8B8ZtvsGTx2L4Rbg+HSAnl8BF5Yseygi5qWfd2fYv5mZTbEsRWF7RPykMBMRdwATXkKKiNuBP4xp+1FE7E5nVwJHTCJWMzOrM0VU7l0q6TKgjeQmcwBvBh4DrgWIiLsqbNsF3BQRx4+z7HvA1RFxZbreOpKzh23ARaWFaMx2/UA/QEdHR/fQ0FDlDCsYHR2lvb296u2bRavkAc6lGbVKHuBcCnp7e1dHRM94y7IUheUVFkdELKiwbRfjFAVJi4Ee4MyICEnPANoj4vfp/YrvAsdFxLZKsfX09MSqVasqxl/J8PAw8+fPr3r7ZtEqeYBzaUatkgc4lwJJZYtClt5HvVUdtXwwi4DXASdFWpEiYhdJl1ciYrWkh4Cjger/4puZ2aQ1dOQ1SacCHwFOT7u6FtqfJ2lWOn0UMBd4uJGxmZlZtldnV0XSVcB84FBJm4CLSXobPQO4RRLAyrSn0auAf5W0G9gDvDsi/jDujs3MrG7qVhQiYuE4zVeUWfda0hvXZmaWnywPr50l6aB0+iJJ10l6af1DMzOzRstyT+FjEbFd0iuAU4BlwFfqG5aZmeUhS1HYk36/FvhKRNwAPL1+IZmZWV6yFIXfSLocOBu4OX2moKG9lszMrDGy/HE/G/ghcGpEPA4cAny4nkGZmVk+KvY+kvQ04M7SJ5IjYjOwud6BmZlZ41U8U4iIvcAvJXnYcTOzGSDLcwqzgXWS7gR2FBoj4vS6RWVmZrnIUhQ+UfcozMysKWR5Id4KSZ3A3Ij4saQ2YFb9QzMzs0bL8kTzO4H/A1yeNh1O8mprMzNrMVm6pJ4HvJxk8Bsi4kHgsHoGZWZm+chSFHZFxBOFGUn7kYzAZmZmLSZLUVgh6aPAMyWdDHwH+F59wzIzszxkKQoXAL8F7gHeBdwMXFTPoMzMLB9Zeh/tlbQM+DnJZaMHYqKBnc3MbFrK0vvotcBDwP8Gvgisl3Rahu2WStoqaW1J2yGSbpH0YPp9cMmyCyWtl/SApFOqS8emq8FB6OqCBQtOpKsrmTezxsty+ejzQG9EzI+IE4Fe4LIM230DOHVM2wXArRExF7g1nUfSscA5wHHpNl8ujNlsrW9wEPr7YcMGiBAbNiTzLgxmjZelKGyNiPUl8w8DWyfaKCJuB8aOs3wGySA9pN9vKGkfiohdEfEIsB44IUNs1gIWL4adO/dt27kzaTezxsrymot1km4GriG5p3AW8AtJZwJExHWTOF5H+pZVImKzpMLzDocDK0vW25S2PYWkfqAfoKOjg+Hh4Ukcfl+jo6M1bd8spnseGzeeCGic9mB4eEXjA5oi0/13KWiVPMC5ZJGlKBwAbAFOTOd/SzKmwutJisRkikI5T/2LUOZZiIgYAAYAenp6Yv78+VUfdHh4mFq2bxbTPY85c5JLR09t17TOa7r/LgWtkgc4lyyy9D56+xQeb4uk2elZwmyevAy1CTiyZL0jgEen8LjWxJYsSe4hlF5CamtL2s2ssbL0PjpA0nmSvpz2KFoqaWmVx7sRWJROLwJuKGk/R9IzJL0AmAvcWeUxbJrp64OBAejsBCno7Ezm+/ryjsxs5slyo/lbwPOBU4AVJP+K3z7RRpKuAn4GHCNpk6R3AJ8CTpb0IHByOk9ErCO5Z3Ev8APgvIjYM/l0bLrq64OREbjtthWMjLggmOUlyz2Fv46IsySdERHLJH2bZMzmiiJiYZlFJ5VZfwngCwZmZjnKcqbwl/T7cUnHA88GuuoWkZmZ5SbLmcJA+uTxx0iu/ben02Zm1mKy9D76ejq5AjiqvuGYmVmesvQ+eq6kL0i6S9JqSf9T0nMbEZyZmTVWlnsKQyTPE7wJ+Cfgd8DV9QzKzMzykeWewiER8cmS+X+T9IY6xWNmZjnKcqawXNI5kp6Wfs4Gvl/vwMzMrPGyFIV3Ad8GdqWfIeCDkrZL2lbP4MzMrLGy9D46qBGBmJlZ/rKcKZiZ2QzhomBmZkUuCmZmVpSpKEh6haS3p9PPS19vbWZmLSbLE80XAx8BLkyb9geurGdQZmaWjyxnCm8ETgd2AETEo4B7JJmZtaAsReGJiAjSMZMlHVjfkMzMLC9ZXnNxjaTLgedIeidwLvC1ag8o6Rj2fXfSUcDHgecA7wR+m7Z/NCJurvY4ZmY2eVkeXvucpJOBbcAxwMcj4pZqDxgRDwDzACTNAn4DXA+8HbgsIj5X7b7NzKw2Wc4USItA1YWggpOAhyJig6Q67N7MzCZDye2CCitI20nvJ5T4I7AK+FBEPFz1waWlwF0R8UVJlwBvIzkjKez7sXG26Qf6ATo6OrqHhoaqPTyjo6O0t7dXvX2zaJU8wLk0o1bJA5xLQW9v7+qI6Bl3YURU/ACfIHkp3kHAs0j+IH8ceDMwPNH2Ffb7dJKxGTrS+Q5gFsnN7yXA0on20d3dHbVYvnx5Tds3i1bJI8K5NKNWySPCuRQAq6LM39UsvY9OjYjLI2J7RGyLiAHgNRFxNXBwVWUqcRrJWcKWtDhtiYg9EbGX5Eb2CTXs28zMqpClKOyVdPaY8RQKKl97qmwhcFVhRtLskmVvBNbWsG8zM6tClhvNfcD/Ar5MUgRWAm+R9EzgvdUcVFIbcDLJZamCz0ialx5jZMwyMzNrgCxdUh8GXl9m8R3VHDQidgLPHdP21mr2ZWZmU2fCoiDpAOAdwHHAAYX2iDi3jnGZmVkOstxT+BbwfOAUYAVwBLC9nkGZmVk+shSFv46IjwE7ImIZ8FrgRfUNy8zM8pClKPwl/X5c0vHAs4GuukVkZma5ydL7aEDSwcBFwI1AO/CxukZlZma5qFgUJD0N2BbJ6yZuJ3mjqZmZtaiKl4/Sp4urehbBzMymnyz3FG6R9C+SjpR0SOFT98jMzKzhstxTKDyPcF5JW+BLSWZmLSfLE80vaEQgZmaWvwkvH0lqk3SRpIF0fq6k19U/NDMza7Qs9xT+A3gC+Id0fhPwb3WLyDIbHISuLliw4ES6upJ5M7NaZCkKL4yIz5A+xBYRfwI8dmbOBgehvx82bIAIsWFDMu/CYGa1yFIUnkhfkx0Akl4I7KprVDahxYth585923buTNrNzKqVpffRJcAPgCMlDQIvJxlL2XK0cePk2s3MssjS++hHklYDLyO5bHR+RPyuloNKGiF50+oeYHdE9KTPPlxN8l6lEeDs9ElqG8ecOcmlo/HazcyqlaX30Y3Aq4HhiLip1oJQojci5kVETzp/AXBrRMwFbk3nrYwlS6Ctbd+2trak3cysWlnuKXweeCVwr6TvSPqndOCdqXYGsCydXga8oQ7HaBl9fTAwAJ2dIAWdncl8X1/ekZnZdKaIyLaiNAtYALwTODUinlX1QaVHgMdIbl5fHhEDkh6PiOeUrPNYRBw8zrb9QD9AR0dH99DQULVhMDo6Snt7e9XbN4tWyQOcSzNqlTzAuRT09vauLrlKs6+ImPADPBM4G7gWeAT4QpbtKuzvr9Lvw4BfAq8CHh+zzmMT7ae7uztqsXz58pq2bxatkkeEc2lGrZJHhHMpAFZFmb+rWcZovhr4e5IeSF8iubewt6ry9GQhejT93irpeuAEYIuk2RGxWdJsYGstxzAzs8nL+kTzCyPi3RFxG/BfJX2p2gNKOlDSQYVpkpvYa0kG8FmUrrYIuKHaY5iZWXWydEn9gaR5khYCbya5fHRdDcfsAK6XVDj+t9Nj/AK4RtI7gI3AWTUcw8zMqlC2KEg6GjgHWAj8nuQZAkVEby0HjIiHgReP0/574KRa9m1mZrWpdKZwP/AT4PURsR5A0gcaEpWZmeWi0j2FNwH/D1gu6WuSTsIvwjMza2lli0JEXB8Rbwb+BhgGPgB0SPqKpFc3KD4zM2ugCXsfRcSOiBiMiNcBRwBrmOavoPA4BGZm48vyltSiiPgDcHn6mZYK4xAkr51+chwC8CsizMyyPKfQUjwOgZlZeTOuKHgcAjOz8mZcUSg33oDHITAzm4FFweMQmJmVN+OKgschMDMrb8YVBUgKwMgI3HbbCkZGXBDMzApmZFEwM7PxuSiYmVmRi4KZmRW5KJiZWZGLgpmZFTW8KEg6UtJySfdJWifp/LT9Ekm/kbQm/bym0bGZmc10k3oh3hTZDXwoIu5Kx2peLemWdNllEfG5HGIyMzNyKAoRsRnYnE5vl3QfcHij4zAzs6dSROR3cKkLuB04Hvgg8DZgG7CK5GzisXG26Qf6ATo6OrqHhoaqPv7o6Cjt7e1Vb98sWiUPcC7NqFXyAOdS0NvbuzoiesZdGBG5fIB2YDVwZjrfAcwiuc+xBFg60T66u7ujFsuXL69p+2bRKnlEOJdm1Cp5RDiXAmBVlPm7mkvvI0n7A9cCgxFxHUBEbImIPRGxF/gacEIesZmZzWR59D4ScAVwX0RcWtI+u2S1NwJrGx2bmdlMl0fvo5cDbwXukbQmbfsosFDSPCCAEeBdOcRmZjaj5dH76A5A4yy6udGxmJnZvvxEs5mZFbkomJlZkYuCmZkVuSiYmVmRi4KZmRW5KJiZWZGLgpmZFbkomJlZkYuCmZkVuSiYmVmRi4KZmRW5KJiZWZGLgpmZFbkomJlZkYuCmZkVuSiYmVlR0xUFSadKekDSekkX5B2PmdlM0lRFQdIs4EvAacCxJEN0HptvVGZmM0dTFQXgBGB9RDwcEU8AQ8AZOcdkZjZjNHyM5gkcDvy6ZH4T8PelK0jqB/rT2VFJD9RwvEOB39WwfbNolTzAuTSjVskDnEtBZ7kFzVYUNE5b7DMTMQAMTMnBpFUR0TMV+8pTq+QBzqUZtUoe4FyyaLbLR5uAI0vmjwAezSkWM7MZp9mKwi+AuZJeIOnpwDnAjTnHZGY2YzTV5aOI2C3pvcAPgVnA0ohYV8dDTsllqCbQKnmAc2lGrZIHOJcJKSImXsvMzGaEZrt8ZGZmOXJRMDOzohlRFCQtlbRV0tox7e9LX6mxTtJn8opvMsbLRdLVktaknxFJa3IMMbMyucyTtDLNZZWkE/KMMYsyebxY0s8k3SPpe5KelWeMWUk6UtJySfel/1+cn7YfIukWSQ+m3wfnHetEKuRyVjq/V1LTd0+tkMdnJd0v6W5J10t6zpQcMCJa/gO8CngpsLakrRf4MfCMdP6wvOOsNpcxyz8PfDzvOGv4XX4EnJZOvwYYzjvOKvP4BXBiOn0u8Mm848yYy2zgpen0QcCvSF458xnggrT9AuDTecdaQy5/CxwDDAM9ecdZQx6vBvZL2z89Vb/JjDhTiIjbgT+MaX4P8KmI2JWus7XhgVWhTC4ASBJwNnBVQ4OqUplcAij8q/rZTIPnVMrkcQxwezp9C/CmhgZVpYjYHBF3pdPbgftI3jRwBrAsXW0Z8IZcApyEcrlExH0RUcubEBqqQh4/iojd6WorSZ7rqtmMKAplHA28UtLPJa2Q9Hd5BzQFXglsiYgH8w6kBu8HPivp18DngAvzDadqa4HT0+mz2PehzGlBUhfwEuDnQEdEbIbkjxRwWI6hTdqYXKatCnmcC/znVBxjJheF/YCDgZcBHwauSf+lPZ0tZJqcJVTwHuADEXEk8AHgipzjqda5wHmSVpOc8j+RczyTIqkduBZ4f0RsyzueWrRKLuXykLQY2A0MTsVxmurhtQbbBFwXyQW5OyXtJXnB1G/zDas6kvYDzgS6846lRouA89Pp7wBfzzGWqkXE/STXfJF0NPDafCPKTtL+JH98BiPiurR5i6TZEbFZ0mxgWlxuLZPLtFMuD0mLgNcBJ6V/y2o2k88UvgssgOL/tE9ner898R+B+yNiU96B1OhR4MR0egEwLS+FSTos/X4acBHw1XwjyiY9W74CuC8iLi1ZdCNJwSb9vqHRsU1WhVymlXJ5SDoV+AhwekTsnLID5n1nvUF3768CNgN/ITlDeAdJEbiS5NrvXcCCvOOsNpe0/RvAu/OObwp+l1cAq4Ffklw37c47zirzOJ+kl8ivgE+Rvj2g2T/pf/8A7gbWpJ/XAM8FbiUp0rcCh+Qdaw25vDH9nXYBW4Af5h1rlXmsJxlqoND21ak4nl9zYWZmRTP58pGZmY3homBmZkUuCmZmVuSiYGZmRS4KZmZW5KJgNglK3CHptJK2syX9IM+4zKaKu6SaTZKk40metn4JybCxa4BTI+KhKvY1KyL2TG2EZtVzUTCrQjr+xg7gwPS7E3gRyatjLomIG9KXl30rXQfgvRHxU0nzgYtJHnibFxHHNjZ6s/JcFMyqIOlAkifhnwBuAtZFxJXpQCd3kpxFBLA3Iv4saS5wVUT0pEXh+8DxEfFIHvGblTOTX4hnVrWI2CHpamCUZAyL10v6l3TxAcAckvc4fVHSPGAPyevaC+50QbBm5KJgVr296UfAm2LMwC2SLiF5t86LSTp1/Llk8Y4GxWg2Ke59ZFa7HwLvK4zHIeklafuzgc0RsRd4K8lNabOm5qJgVrtPAvsDd0tam84DfBlYJGklyaUjnx1Y0/ONZjMzK/KZgpmZFbkomJlZkYuCmZkVuSiYmVmRi4KZmRW5KJiZWZGLgpmZFf1/8/QldI3+odcAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "yr_p = [i for i in range(16,23)]  # year list\n",
    "num_of_months_per_year = [len(df[df['year']==str(yr)]['month'].unique()) for yr in range(16,23)]  # count unique number of months in dataframe per year\n",
    "\n",
    "num_per_year_per_month = [num_per_year[i]/num_of_months_per_year[i] for i in range(len(num_per_year))]  # normalize number of tweets per year per month\n",
    "plt.plot(yr_p,num_per_year_per_month,'bo')\n",
    "plt.xlabel('Year')\n",
    "plt.ylabel('Average papers per month')\n",
    "plt.grid()\n",
    "plt.ylim([0, 220])\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f3ce1285-e6a5-4bf2-9776-e4a7155149b2",
   "metadata": {},
   "source": [
    "Obviously the above metric plot suffers from biases involving query choice, but it indicates the rise in popularity of academics in Twitter writing about their papers. \n",
    "\n",
    "Note, near the end of 2017, Twitter expanded their character limit from 140 to 280 characters per tweet, which could be a contributing factor to the increase."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dd42d049-2a63-4e93-9425-33adbcc8b073",
   "metadata": {},
   "source": [
    "### Investigate reallly long tweets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "82abd906-41c6-4f46-803d-3d272ded3e35",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>TweetID</th>\n",
       "      <th>AuthorID</th>\n",
       "      <th>AuthorName</th>\n",
       "      <th>Tweets</th>\n",
       "      <th>arxiv_link</th>\n",
       "      <th>Abstract</th>\n",
       "      <th>Title</th>\n",
       "      <th>Thread_length</th>\n",
       "      <th>Tweets_coarse</th>\n",
       "      <th>year</th>\n",
       "      <th>month</th>\n",
       "      <th>tweet_length</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2631</th>\n",
       "      <td>35</td>\n",
       "      <td>1196267948056424448</td>\n",
       "      <td>3245949691</td>\n",
       "      <td>Rebecca Leane</td>\n",
       "      <td>New paper! Cooking Pasta with Dark Matter: Kin...</td>\n",
       "      <td>https://arxiv.org/abs/1911.06334</td>\n",
       "      <td>Neutron stars serve as excellent next-generati...</td>\n",
       "      <td>Warming Nuclear Pasta with Dark Matter: Kineti...</td>\n",
       "      <td>36</td>\n",
       "      <td>['New paper!\\n\\nCooking Pasta with Dark Matter...</td>\n",
       "      <td>19</td>\n",
       "      <td>11</td>\n",
       "      <td>8981</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4031</th>\n",
       "      <td>89</td>\n",
       "      <td>1212988781537554432</td>\n",
       "      <td>2416760538</td>\n",
       "      <td>Peter Gao</td>\n",
       "      <td>New paper! \"Deflating Super-Puffs: Impact of P...</td>\n",
       "      <td>https://arxiv.org/abs/2001.00055</td>\n",
       "      <td>The observed mass-radius relationship of low-m...</td>\n",
       "      <td>Deflating Super-Puffs: Impact of Photochemical...</td>\n",
       "      <td>45</td>\n",
       "      <td>['New paper! \"Deflating Super-Puffs: Impact of...</td>\n",
       "      <td>20</td>\n",
       "      <td>01</td>\n",
       "      <td>9857</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4300</th>\n",
       "      <td>241</td>\n",
       "      <td>1311851889009418241</td>\n",
       "      <td>3245949691</td>\n",
       "      <td>Rebecca Leane</td>\n",
       "      <td>New paper! Exoplanets as New Sub-GeV Dark Matt...</td>\n",
       "      <td>https://arxiv.org/abs/2010.00015</td>\n",
       "      <td>We present exoplanets as new targets to discov...</td>\n",
       "      <td>Exoplanets as Sub-GeV Dark Matter Detectors</td>\n",
       "      <td>46</td>\n",
       "      <td>['New paper!\\n\\nExoplanets as New Sub-GeV Dark...</td>\n",
       "      <td>20</td>\n",
       "      <td>10</td>\n",
       "      <td>9231</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4390</th>\n",
       "      <td>5</td>\n",
       "      <td>1334191213306654720</td>\n",
       "      <td>143430352</td>\n",
       "      <td>Rachael Meager</td>\n",
       "      <td>Hello! Tamara Broderick, Ryan Giordano and I h...</td>\n",
       "      <td>https://arxiv.org/abs/2011.14999</td>\n",
       "      <td>We propose a method to assess the sensitivity ...</td>\n",
       "      <td>An Automatic Finite-Sample Robustness Metric: ...</td>\n",
       "      <td>71</td>\n",
       "      <td>['Hello! Tamara Broderick, Ryan Giordano and I...</td>\n",
       "      <td>20</td>\n",
       "      <td>11</td>\n",
       "      <td>12188</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4544</th>\n",
       "      <td>159</td>\n",
       "      <td>1332290582778142720</td>\n",
       "      <td>1212040530592333826</td>\n",
       "      <td>Sunny Vagnozzi</td>\n",
       "      <td>A few days ago, with Avi Loeb from Harvard (th...</td>\n",
       "      <td>https://arxiv.org/abs/2011.11645</td>\n",
       "      <td>The question of whether Cosmic Microwave Backg...</td>\n",
       "      <td>Eppur \\`e piatto? The cosmic chronometer take ...</td>\n",
       "      <td>43</td>\n",
       "      <td>['A few days ago, with Avi Loeb from Harvard (...</td>\n",
       "      <td>20</td>\n",
       "      <td>11</td>\n",
       "      <td>10386</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4640</th>\n",
       "      <td>68</td>\n",
       "      <td>1339231323995713537</td>\n",
       "      <td>4365927557</td>\n",
       "      <td>Dr. Jake Turner 🌅</td>\n",
       "      <td>***New 1st author paper ***** \"The search for ...</td>\n",
       "      <td>http://arxiv.org/abs/2012.07926</td>\n",
       "      <td>Observing planetary auroral radio emission is ...</td>\n",
       "      <td>The search for radio emission from the exoplan...</td>\n",
       "      <td>31</td>\n",
       "      <td>['***New 1st author paper *****\\n\\n\"The search...</td>\n",
       "      <td>20</td>\n",
       "      <td>12</td>\n",
       "      <td>9509</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4641</th>\n",
       "      <td>69</td>\n",
       "      <td>1339219770730799106</td>\n",
       "      <td>3308167500</td>\n",
       "      <td>David C. Norris, MD 🌻</td>\n",
       "      <td>NEW #DTAT paper: What Were They Thinking? Phar...</td>\n",
       "      <td>https://arxiv.org/abs/2012.05301</td>\n",
       "      <td>If explicit, formal consideration of clinical ...</td>\n",
       "      <td>What Were They Thinking? Pharmacologic priors ...</td>\n",
       "      <td>39</td>\n",
       "      <td>['NEW #DTAT paper: What Were They Thinking? Ph...</td>\n",
       "      <td>20</td>\n",
       "      <td>12</td>\n",
       "      <td>8580</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4917</th>\n",
       "      <td>147</td>\n",
       "      <td>1234301978156634112</td>\n",
       "      <td>3245949691</td>\n",
       "      <td>Rebecca Leane</td>\n",
       "      <td>Galactic Center Excess update! &lt;LINK&gt; &lt;LINK&gt; W...</td>\n",
       "      <td>https://arxiv.org/abs/2002.12370</td>\n",
       "      <td>We re-examine evidence that the Galactic Cente...</td>\n",
       "      <td>Spurious Point Source Signals in the Galactic ...</td>\n",
       "      <td>46</td>\n",
       "      <td>['Galactic Center Excess update! \\n&lt;LINK&gt;\\n&lt;LI...</td>\n",
       "      <td>20</td>\n",
       "      <td>02</td>\n",
       "      <td>9756</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5090</th>\n",
       "      <td>4</td>\n",
       "      <td>1259420853026119682</td>\n",
       "      <td>2603024598</td>\n",
       "      <td>Ricardo Pérez-Marco</td>\n",
       "      <td>New paper \"The Cantor Riemannium\". First examp...</td>\n",
       "      <td>https://arxiv.org/abs/2004.10541</td>\n",
       "      <td>The Riemann surface of a holomorphic germ is t...</td>\n",
       "      <td>The Cantor Riemannium</td>\n",
       "      <td>38</td>\n",
       "      <td>['New paper \"The Cantor Riemannium\". First exa...</td>\n",
       "      <td>20</td>\n",
       "      <td>04</td>\n",
       "      <td>9131</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5139</th>\n",
       "      <td>53</td>\n",
       "      <td>1255095770627428352</td>\n",
       "      <td>3308167500</td>\n",
       "      <td>David C. Norris, MD 🌻</td>\n",
       "      <td>NEW WORKING PAPER @arxiv … A retrospective, mo...</td>\n",
       "      <td>https://arxiv.org/abs/2004.12755</td>\n",
       "      <td>The commonplace description of phase 1 clinica...</td>\n",
       "      <td>Retrospective analysis of a fatal dose-finding...</td>\n",
       "      <td>52</td>\n",
       "      <td>['NEW WORKING PAPER @arxiv …\\n\\nA retrospectiv...</td>\n",
       "      <td>20</td>\n",
       "      <td>04</td>\n",
       "      <td>11021</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6368</th>\n",
       "      <td>12</td>\n",
       "      <td>1356057142709477382</td>\n",
       "      <td>3245949691</td>\n",
       "      <td>Rebecca Leane</td>\n",
       "      <td>New paper! Celestial-Body Focused Dark Matter ...</td>\n",
       "      <td>https://arxiv.org/abs/2101.12213</td>\n",
       "      <td>Indirect detection experiments typically measu...</td>\n",
       "      <td>Celestial-Body Focused Dark Matter Annihilatio...</td>\n",
       "      <td>39</td>\n",
       "      <td>['New paper!\\n\\nCelestial-Body Focused Dark Ma...</td>\n",
       "      <td>21</td>\n",
       "      <td>01</td>\n",
       "      <td>8489</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      Unnamed: 0              TweetID             AuthorID  \\\n",
       "2631          35  1196267948056424448           3245949691   \n",
       "4031          89  1212988781537554432           2416760538   \n",
       "4300         241  1311851889009418241           3245949691   \n",
       "4390           5  1334191213306654720            143430352   \n",
       "4544         159  1332290582778142720  1212040530592333826   \n",
       "4640          68  1339231323995713537           4365927557   \n",
       "4641          69  1339219770730799106           3308167500   \n",
       "4917         147  1234301978156634112           3245949691   \n",
       "5090           4  1259420853026119682           2603024598   \n",
       "5139          53  1255095770627428352           3308167500   \n",
       "6368          12  1356057142709477382           3245949691   \n",
       "\n",
       "                 AuthorName  \\\n",
       "2631          Rebecca Leane   \n",
       "4031              Peter Gao   \n",
       "4300          Rebecca Leane   \n",
       "4390         Rachael Meager   \n",
       "4544         Sunny Vagnozzi   \n",
       "4640      Dr. Jake Turner 🌅   \n",
       "4641  David C. Norris, MD 🌻   \n",
       "4917          Rebecca Leane   \n",
       "5090    Ricardo Pérez-Marco   \n",
       "5139  David C. Norris, MD 🌻   \n",
       "6368          Rebecca Leane   \n",
       "\n",
       "                                                 Tweets  \\\n",
       "2631  New paper! Cooking Pasta with Dark Matter: Kin...   \n",
       "4031  New paper! \"Deflating Super-Puffs: Impact of P...   \n",
       "4300  New paper! Exoplanets as New Sub-GeV Dark Matt...   \n",
       "4390  Hello! Tamara Broderick, Ryan Giordano and I h...   \n",
       "4544  A few days ago, with Avi Loeb from Harvard (th...   \n",
       "4640  ***New 1st author paper ***** \"The search for ...   \n",
       "4641  NEW #DTAT paper: What Were They Thinking? Phar...   \n",
       "4917  Galactic Center Excess update! <LINK> <LINK> W...   \n",
       "5090  New paper \"The Cantor Riemannium\". First examp...   \n",
       "5139  NEW WORKING PAPER @arxiv … A retrospective, mo...   \n",
       "6368  New paper! Celestial-Body Focused Dark Matter ...   \n",
       "\n",
       "                            arxiv_link  \\\n",
       "2631  https://arxiv.org/abs/1911.06334   \n",
       "4031  https://arxiv.org/abs/2001.00055   \n",
       "4300  https://arxiv.org/abs/2010.00015   \n",
       "4390  https://arxiv.org/abs/2011.14999   \n",
       "4544  https://arxiv.org/abs/2011.11645   \n",
       "4640   http://arxiv.org/abs/2012.07926   \n",
       "4641  https://arxiv.org/abs/2012.05301   \n",
       "4917  https://arxiv.org/abs/2002.12370   \n",
       "5090  https://arxiv.org/abs/2004.10541   \n",
       "5139  https://arxiv.org/abs/2004.12755   \n",
       "6368  https://arxiv.org/abs/2101.12213   \n",
       "\n",
       "                                               Abstract  \\\n",
       "2631  Neutron stars serve as excellent next-generati...   \n",
       "4031  The observed mass-radius relationship of low-m...   \n",
       "4300  We present exoplanets as new targets to discov...   \n",
       "4390  We propose a method to assess the sensitivity ...   \n",
       "4544  The question of whether Cosmic Microwave Backg...   \n",
       "4640  Observing planetary auroral radio emission is ...   \n",
       "4641  If explicit, formal consideration of clinical ...   \n",
       "4917  We re-examine evidence that the Galactic Cente...   \n",
       "5090  The Riemann surface of a holomorphic germ is t...   \n",
       "5139  The commonplace description of phase 1 clinica...   \n",
       "6368  Indirect detection experiments typically measu...   \n",
       "\n",
       "                                                  Title  Thread_length  \\\n",
       "2631  Warming Nuclear Pasta with Dark Matter: Kineti...             36   \n",
       "4031  Deflating Super-Puffs: Impact of Photochemical...             45   \n",
       "4300        Exoplanets as Sub-GeV Dark Matter Detectors             46   \n",
       "4390  An Automatic Finite-Sample Robustness Metric: ...             71   \n",
       "4544  Eppur \\`e piatto? The cosmic chronometer take ...             43   \n",
       "4640  The search for radio emission from the exoplan...             31   \n",
       "4641  What Were They Thinking? Pharmacologic priors ...             39   \n",
       "4917  Spurious Point Source Signals in the Galactic ...             46   \n",
       "5090                              The Cantor Riemannium             38   \n",
       "5139  Retrospective analysis of a fatal dose-finding...             52   \n",
       "6368  Celestial-Body Focused Dark Matter Annihilatio...             39   \n",
       "\n",
       "                                          Tweets_coarse year month  \\\n",
       "2631  ['New paper!\\n\\nCooking Pasta with Dark Matter...   19    11   \n",
       "4031  ['New paper! \"Deflating Super-Puffs: Impact of...   20    01   \n",
       "4300  ['New paper!\\n\\nExoplanets as New Sub-GeV Dark...   20    10   \n",
       "4390  ['Hello! Tamara Broderick, Ryan Giordano and I...   20    11   \n",
       "4544  ['A few days ago, with Avi Loeb from Harvard (...   20    11   \n",
       "4640  ['***New 1st author paper *****\\n\\n\"The search...   20    12   \n",
       "4641  ['NEW #DTAT paper: What Were They Thinking? Ph...   20    12   \n",
       "4917  ['Galactic Center Excess update! \\n<LINK>\\n<LI...   20    02   \n",
       "5090  ['New paper \"The Cantor Riemannium\". First exa...   20    04   \n",
       "5139  ['NEW WORKING PAPER @arxiv …\\n\\nA retrospectiv...   20    04   \n",
       "6368  ['New paper!\\n\\nCelestial-Body Focused Dark Ma...   21    01   \n",
       "\n",
       "      tweet_length  \n",
       "2631          8981  \n",
       "4031          9857  \n",
       "4300          9231  \n",
       "4390         12188  \n",
       "4544         10386  \n",
       "4640          9509  \n",
       "4641          8580  \n",
       "4917          9756  \n",
       "5090          9131  \n",
       "5139         11021  \n",
       "6368          8489  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['tweet_length'] = df['Tweets'].apply(lambda x: len(x))\n",
    "\n",
    "df.loc[df['tweet_length']>8000]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c2fd7d8a-5b3a-4b4f-89ca-d2cdcc7baa98",
   "metadata": {},
   "source": [
    "Apparently these threads are legit -- some authors are writing ~30 tweets or 10000 characters, roughly 2 two-column journal pages to my estimates -- A LOT!"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6d83b28e-3668-496e-b5c8-da06f4b8ea6d",
   "metadata": {},
   "source": [
    "### How many \"reasonably-sized\" tweets?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "f508d5d6-3c30-42ea-99b2-0e32d87a82ac",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4942"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "min_chars = 250\n",
    "max_chars = 2500\n",
    "df_mid = df.loc[df['tweet_length'] > min_chars].loc[df['tweet_length'] < max_chars]\n",
    "df_mid.drop(['Unnamed: 0'], axis=1, inplace=True)\n",
    "df_mid.reset_index()\n",
    "\n",
    "len(df_mid)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a46df7db-0762-4727-a404-d5600bec101f",
   "metadata": {},
   "source": [
    "We find ~5000 tweets with characters count between 250 and 2500, roughly 50 to 500 tokens.\n",
    "\n",
    "These cuts eliminate many tweets that write little on the paper or a lot on the paper, potentially surpassing an abstract's length."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "44879c7c-2a3f-4126-ae6c-b54e8c643617",
   "metadata": {},
   "source": [
    "#### Save \"reasonably-sized\" tweets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "3625b279-670d-42f5-a481-e0ee57762bed",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_mid.to_csv('data/data_mid.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3840c016-48e6-44c3-8782-531918ff2040",
   "metadata": {},
   "source": [
    "### Train-Val-Test split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "7cb867b2-dc6b-446c-a69e-b1d0f497caf6",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "df_mid_train, df_mid_rest = train_test_split(df_mid, test_size=0.2)\n",
    "df_mid_val, df_mid_test = train_test_split(df_mid_rest, test_size=0.5)\n",
    "\n",
    "df_mid_train.reset_index(inplace=True, drop=True)\n",
    "df_mid_val.reset_index(inplace=True, drop=True)\n",
    "df_mid_test.reset_index(inplace=True, drop=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "8257abc2-ace7-47ac-820b-6a93de6beaa8",
   "metadata": {},
   "outputs": [],
   "source": [
    "# save sub-datasets\n",
    "df_mid_train.to_csv('data/data_mid_train.csv')\n",
    "df_mid_val.to_csv('data/data_mid_val.csv')\n",
    "df_mid_test.to_csv('data/data_mid_test.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f129c5d2-fd3c-48cc-abc2-af9d90b21b28",
   "metadata": {},
   "source": [
    "#### Save small dataset for debugging"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "a7ec9cbb-0512-4bb4-83cf-622a16ea487c",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_mid_most, df_mid_debug = train_test_split(df_mid, test_size=0.07)\n",
    "\n",
    "df_mid_debug_train, df_mid_debug_rest = train_test_split(df_mid_debug, test_size=0.2)\n",
    "df_mid_debug_val, df_mid_debug_test = train_test_split(df_mid_debug_rest, test_size=0.5)\n",
    "\n",
    "df_mid_debug_train.reset_index(inplace=True, drop=True)\n",
    "df_mid_debug_val.reset_index(inplace=True, drop=True)\n",
    "df_mid_debug_test.reset_index(inplace=True, drop=True)\n",
    "\n",
    "df_mid_debug_train.to_csv('data/data_mid_debug_train.csv')\n",
    "df_mid_debug_val.to_csv('data/data_mid_debug_val.csv')\n",
    "df_mid_debug_test.to_csv('data/data_mid_debug_test.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "968fd98e",
   "metadata": {},
   "source": [
    "# Sentiment"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a68f072a",
   "metadata": {},
   "source": [
    "Let us conduct an exploratory sentiment analysis with a built-in took of NLTK."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "9ece487d",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "df = pd.read_csv('data/full_data.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "6054368d",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package vader_lexicon to\n",
      "[nltk_data]     /Users/nitsanbar/nltk_data...\n",
      "[nltk_data]   Package vader_lexicon is already up-to-date!\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import nltk\n",
    "nltk.download('vader_lexicon')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "b5545358",
   "metadata": {},
   "outputs": [],
   "source": [
    "# we define a minimal procedure to remove \"bad sentences\". Currently, it removes sentences which are very short (< 15 chars) or contain only a particular token (currently '<LINK>')\n",
    "def remove_bad_sentences(list, min_len=15, bad_word_list=[\"<LINK>\"]):\n",
    "    good_sentences = []\n",
    "    for sentence in list:\n",
    "        # if sentence equals a string in list\n",
    "        if sentence in bad_word_list:\n",
    "            continue\n",
    "        if len(sentence) < min_len:\n",
    "            continue\n",
    "        good_sentences.append(sentence)\n",
    "    return good_sentences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "9e7ce217",
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.sentiment import SentimentIntensityAnalyzer\n",
    "from nltk.tokenize import sent_tokenize, word_tokenize\n",
    "\n",
    "sia = SentimentIntensityAnalyzer()\n",
    "sia_scores = []\n",
    "\n",
    "min_num_sentences = 5\n",
    "property = 'pos'\n",
    "sia_prop_moments = []\n",
    "sia_prop = []\n",
    "# run on dataframe of tweets\n",
    "for i in range(len(df)):\n",
    "    tweet = df['Tweets'][i]\n",
    "\n",
    "    # compute overall sia score for the tweet\n",
    "    sia_scores.append(sia.polarity_scores(tweet))\n",
    "    \n",
    "    # tokenize the tweet into sentences, then compute sia score for each sentence and compute the \"moment\" of a property defined above\n",
    "    sent_list = sent_tokenize(tweet)\n",
    "    sent_list = remove_bad_sentences(sent_list)\n",
    "    \n",
    "    # only conduct a sentence-by-sentence analysis if the tweet has at least min_num_sentences\n",
    "    if len(sent_list) >= min_num_sentences:\n",
    "        sum_prop_moment = 0\n",
    "        sum_prop = 0\n",
    "        for i, sent in enumerate(sent_list):\n",
    "            sia_sent = sia.polarity_scores(sent)\n",
    "            sum_prop_moment += sia_sent[property] * i / (len(sent_list) - 1)\n",
    "            sum_prop += sia_sent[property]\n",
    "        if sum_prop_moment == 0 and sum_prop == 0:\n",
    "            prop_moment = 0\n",
    "        else:\n",
    "            prop_moment = sum_prop_moment / sum_prop\n",
    "        sia_prop_moments.append(prop_moment)\n",
    "        sia_prop.append(sum_prop / len(sent_list))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "6cdbe4f1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAEGCAYAAACJnEVTAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAA2SUlEQVR4nO3dd3wUdf7H8ddnd1OA0LsEBQRpKfQOooAUQYogoCjieejpIXrFU+9nOcuJd54FPeVsJ4qCjY4gTYpIkRIgNOkSQUB6T3bm+/tj11yQBEKy2dnyeT4eeezu7JT3JJtPvvnOzHfEGINSSqno4HI6gFJKqeDRoq+UUlFEi75SSkURLfpKKRVFtOgrpVQU8Tgd4FIqVKhgatSo4XQMpZQKK6tWrfrZGFPx19NDvujXqFGDlStXOh1DKaXCiojszm26du8opVQU0aKvlFJRRIu+UkpFkZDv089NVlYWGRkZnD171ukoqojFx8eTmJhITEyM01GUighhWfQzMjIoWbIkNWrUQEScjqOKiDGGQ4cOkZGRQc2aNZ2Oo1RECMvunbNnz1K+fHkt+BFORChfvrz+R6dUAF2y6IvIeyJyQETSc0wrJyJzRGSr/7FsjvceFZFtIrJFRLrmmN5URNb73xsthazYWvCjg/6clQqs/LT03we6/WraI8A8Y0wdYJ7/NSLSABgENPQv84aIuP3LvAkMB+r4v369TqWik205nUBFkUv26RtjFolIjV9N7g109D8fCywA/uKfPsEYcw7YKSLbgBYisgsoZYxZCiAiHwB9gJmF3gOHJCQkcPLkyezX77//PitXruT1119nzJgxFC9enDvuuCPXZRcsWEBsbCxt2rQJVlwVSvatg3WfwP4NcGATnDoAZWtC5QZQNRWa3AkJF1xIqVRAFPRAbmVjzD4AY8w+Eankn14NWJZjvgz/tCz/819Pz5WIDMf3XwFXXnllASM65957773o+wsWLCAhISEgRd+yLNxu96VnVM47uAW+fg42TgFPPFSsB1dfDyWrwKFtvj8Am6bD4peh1b3QZgQUK3vp9Sp1GQJ9IDe3Dlhzkem5Msa8ZYxpZoxpVrFi+LV4nnrqKV588UUARo8eTYMGDUhJSWHQoEHs2rWLMWPG8PLLL9OoUSMWL17M7t276dSpEykpKXTq1IkffvgBgO3bt9OqVSuaN2/OE088QUJCAuD7o3Hddddx6623kpycDECfPn1o2rQpDRs25K233srOkpCQwF/+8heaNm1K586dWbFiBR07dqRWrVpMnTo1yN+ZKGXbMP85eKMVbJsHHR6GP26BexZC3zeh85Mw8EMYsRJ+/x3U7QaL/wWvpsL3s51OryJMQVv6+0Wkqr+VXxU44J+eAVTPMV8isNc/PTGX6YU38xH4aX1AVpWtSjJ0H3XRWc6cOUOjRo2yXx8+fJibbrrpgvlGjRrFzp07iYuL4+jRo5QpU4Z7772XhIQE/vSnPwHQq1cv7rjjDoYOHcp7773HAw88wOTJkxk5ciQjR45k8ODBjBkz5rz1rlixgvT09OxTGd977z3KlSvHmTNnaN68OTfffDPly5fn1KlTdOzYkRdeeIG+ffvyf//3f8yZM4eNGzcydOjQXDOrAMo6C5N/BxsmQuqtcMOzUKJ83vNXqAP934N2f/AtN34gdHsBWg4PXmYV0Qra0p8KDPU/HwpMyTF9kIjEiUhNfAdsV/i7gk6ISCv/WTt35FgmLBUrVoy0tLTsr6effjrX+VJSUrjtttsYN24cHk/uf2OXLl3KrbfeCsDtt9/ON998kz19wIABANnv/6JFixbnnbs+evRoUlNTadWqFXv27GHr1q0AxMbG0q2b75h5cnIy1157LTExMSQnJ7Nr166CfwPUpZ36Gcb28hX8Lk9DnzcuXvBzqpIEw2bCNd1g5p9h5l/0gK8KiEu29EVkPL6DthVEJAN4EhgFfCoivwF+AAYAGGM2iMinwEbAC9xvjPnlk/o7fGcCFcN3ADcwB3Ev0SJ32owZM1i0aBFTp07lmWeeYcOGDZdcJj+nKZYoUSL7+YIFC5g7dy5Lly6lePHidOzYMfvc9piYmOz1uVwu4uLisp97vd6C7JLKj8zT8FF/Xz/9LR9CgwL8RxWXAAPHwezHYdm/weWBrs8FPquKKvk5e2dwHm91ymP+54ALPpnGmJVA0mWlC3O2bbNnzx6uu+462rVrx8cff8zJkycpWbIkx48fz56vTZs2TJgwgdtvv52PPvqIdu3aAdCqVSu++OILBg4cyIQJE/LczrFjxyhbtizFixdn8+bNLFu2LM95VRDYNky6B/amweDxULd7wdflckO3v4PthaWvQ/na0GxYwKKq6BOWV+SGC8uyGDJkCMnJyTRu3JiHHnqIMmXK0KtXLyZNmpR9IHf06NH897//JSUlhQ8//JBXX30VgFdeeYWXXnqJFi1asG/fPkqXLp3rdrp164bX6yUlJYXHH3+cVq1aBXM31a/Nfxo2TfW1ygtT8HPq+neo3QVm/BG2fx2YdaqoJMbkeRJNSGjWrJn59U1UNm3aRP369R1KFDynT5+mWLFiiAgTJkxg/PjxTJkS1odCCiSsft5rP4FJw6HZXXDjSxDIK4rPHof3usGxDBj+NZS/OnDrVhFHRFYZY5r9erq29EPYqlWraNSoESkpKbzxxhv861//cjqSupije3wt8avaQvd/BLbgA8SXgls/8a138u/0wK4qkLAcZTNatG/fnrVr1zodQ+WHMTD194CBPm+Cu4iGgi5THXr8Eyb+Fpb+G9o+UDTbURFLW/pKBcLKd2HHArjhGSh7VdFuK3kA1OsJ85+FA5uLdlsq4mjRV6qwDu+E2U/4hlRoGoQza0Sg58sQW8LXzWPpqbcq/7ToK1VYM/7oO7XyptcC34+fl4RK0PMl2LsaVrx16fmV8tOir1RhbJ0D2+dBx0egdOKl5w+kBn18/10sHAWnDwd32yps6YHcAtq/fz8PPfQQy5Yto2zZssTGxvLwww/Tt2/fItvmypUr+eCDDxg9enSRbSMS1XhkRpGs143FrNhHcFOFrlMSyZoSmO3sGnVj/mYUgRuegzFtYeEL0P2FgGxfRTZt6ReAMYY+ffrQoUMHduzYwapVq5gwYQIZGRmXXrgQmjVrpgU/hAx2z6eO60ee995KllPtp8oNoOmd8N078PNWZzKosKJFvwDmz59PbGzseePmX3XVVYwYMYJdu3bRvn17mjRpQpMmTfj2228B3/g4PXv2zJ7/97//Pe+//z4AjzzySPbwy7+MvPnZZ5+RlJREamoqHTp0uGAdK1asoE2bNjRu3Jg2bdqwZcsWwHczl379+tGtWzfq1KnDww8/XOTfj2hUilP8wfMZ31oNmGM3dTbMdX+FmOIw+/+czaHCgnbvFMCGDRto0qRJru9VqlSJOXPmEB8fz9atWxk8eDC/vqI4p8OHDzNp0iQ2b96MiHD06FEAnn76ab766iuqVauWPS2nevXqsWjRIjweD3PnzuWxxx7jiy++ACAtLY01a9YQFxdH3bp1GTFiBNWrV79gHarg7vNMoQyneNY7hNxvFxFEJSpAhz/BnCdgx0Koda2zeVRI05Z+ANx///2kpqbSvHlzsrKy+O1vf0tycjIDBgxg48aNF122VKlSxMfHc/fddzNx4kSKFy8OQNu2bbnzzjt5++23sawLr7w8duwYAwYMICkpiYceeui80Ts7depE6dKliY+Pp0GDBuzevTuwOxzlynOMoe7ZTLLbstHUcDqOT8t7oVQ1WPC870IxpfKgRb8AGjZsyOrVq7Nf//vf/2bevHkcPHiQl19+mcqVK7N27VpWrlxJZmYmAB6PB9u2s5f5Zehjj8fDihUruPnmm5k8eXL22Pdjxozh2WefZc+ePTRq1IhDhw6dl+Hxxx/nuuuuIz09nWnTpmWvD8gePhnA7XbrEMoB9lvPl8SSxeveojtof9k8cdDuIfhhKexa7HQaFcK06BfA9ddfz9mzZ3nzzTezp50+fRrwtcCrVq2Ky+Xiww8/zG6lX3XVVWzcuJFz585x7Ngx5s2bB8DJkyc5duwYPXr04JVXXiEtLQ3w3SqxZcuWPP3001SoUIE9e/acl+HYsWNUq+a7zfAvxwZU0SvLcW53z2aq3YadpqrTcc7X+HYoWRUW6Fk8Km9a9AtARJg8eTILFy6kZs2atGjRgqFDh/LCCy9w3333MXbsWFq1asX333+ffbOT6tWrc8stt2TfSatx48YAnDhxgp49e5KSksK1117Lyy+/DMCf//xnkpOTSUpKokOHDqSmpp6X4eGHH+bRRx+lbdu2uXb/qKJxt+dLipHJ694+Tke5UEw8tH0Qdn8Du75xOo0KUTq0sgp5hf15B+o8/dKcZEncA3xtN2JEVtENdJbv8/Rzk3XGd0P1inVh6LTAhVJhJ6+hlSPi7J1AX3xTqF86FbHu8swkQc4y2tvP6Sh5iykGbUfCV4/B7qVwVWunE6kQo907SuVDMc5yh3sOs6zmbDVBHm7hcjUdBsXKwbevOZ1EhSAt+krlw83uxZSVk7zt7eF0lEuLLQ7NfwNbvoRD251Oo0KMFn2lLkGwGeaeRZpdi1XmGqfj5E/zu8HlgeVjnE6iQowWfaUu4TpXGle79vGutweOX32bXyWr+G62smYcnDnidBoVQrToh4BZs2ZRt25dateuzahRo/Kcr0aNGiQnJ9OoUSOaNfvfQfmjR4/Sv39/6tWrR/369Vm6dGkwYuc791133UWlSpVISkq64D3LsmjcuPF54xKFmrvdX7LXlGOm3cLpKJen9X2QdRpWve90EhVCtOg7zLIs7r//fmbOnMnGjRsZP378RYdu+Prrr0lLSztvPJ+RI0fSrVs3Nm/ezNq1a4NyOuvl5L7zzjuZNWtWru+9+uqrIX36bQPZRRv3Rt73dsUbbie7VUmGmh1g+VtgZTmdRoWIMPsU586pUyw7duzIf/7zH+rWrcuhQ4e49tprSU9Pv6x1rFixgtq1a1OrVi0ABg0axJQpU2jQoEG+lj9+/DiLFi3Kvio3NjaW2NjYiy6zdu1aRowYwc8//8zmzZsxxvDEE0/wt7/9rUhyd+jQgV27dl0wPSMjgxkzZvDXv/6Vl156Kd/bDqa7PLM4ZeKYYF3vdJSCaXU/jB8IGyZDygCn06gQEBFF3ynbtm2jTp06AKxbt47k5OTz3m/fvj0nTpy4YLkXX3yRzp07A/Djjz+eNwJmYmIiy5cvz3V7IsINN9yAiHDPPfcwfPhwduzYQcWKFRk2bBhr166ladOmvPrqq9lXAv/a2bNnGThwIB988AEtWrTg8ccf5+zZszz11FNFljsvDz74IP/4xz9y3VYoKMMJermW8onVkePk/v0MeXVugHK1fDdu16Kv0KJfYLt376ZatWq4XL4esnXr1pGSknLePIsXX3rgq9yuiJY87rO6ZMkSrrjiCg4cOECXLl2oV68exYsXZ/Xq1bz22mu0bNmSkSNHMmrUKJ555plc1zF37lyaNGlCixa+/umUlBRmzZp13jYDnTs306dPp1KlSjRt2pQFCxbke7lg6u9eRJxk8ZHVyekoBedy+c7bn/M47N8AlRs6nUg5TIt+AaWlpZ1X5FetWsXAgQPPmyc/LebExMTzBlPLyMjgiiuuyHWbv0yvVKkSffv2ZcWKFQwZMoTExERatmwJQP/+/S96UDU9Pf28/0hWr159wb0BAp07N0uWLGHq1Kl8+eWXnD17luPHjzNkyBDGjRuX73UUJcHmVvc8vrOvYYu5MqjbDvQV5mWpyLK4GCa89iRPeocFdN169Xr40aJfQGvXrs0eznjr1q1MmTKFZ5999rx58tNibt68OVu3bmXnzp1Uq1aNCRMm8PHHH18w36lTp7Btm5IlS3Lq1Clmz57NE088QZUqVahevTpbtmyhbt26zJs377x+9U6dOvHBBx9kj8hZvnx55s+fD8D333/PxIkTs+/uVRS58/L888/z/PPPA747gr344oshU/AB2rg2UMv1E6MzQ3jIhXw6Qilm2C3p5/6GF7yDOU2805GUg/TsnQJKS0vDtm1SU1N5+umnqV+/PmPHjr3s9Xg8Hl5//XW6du1K/fr1ueWWW2jY8H//gvfo0YO9e/eyf/9+2rVrR2pqKi1atODGG2/MHnv/tdde47bbbiMlJYW0tDQee+wxAGzbZtu2bZQrVy57fYMHD+bkyZMkJSUxfPhwxo8fT/ny5Yss9y/bbN26NVu2bCExMZF33333srcXbLe553HYJITfaZp5+MjbiZJyhpvc3156ZhXRdJTNAqpduzZr1qyhZMmSjua4mPT0dN57772QPTMmv4I9ymYljvBt3AjetbrzvPe2Am83tBhmxj6ChZuemc8RqIvMtHsndOU1yqa29AvgxIkTuFyukC74AElJSWFf8J0w0P01HrEZH66naeZK+MjqTJJrFymyw+kwykGFKvoi8pCIbBCRdBEZLyLxIlJOROaIyFb/Y9kc8z8qIttEZIuIdC18fGeULFmS77//3ukYqgi4sBnoWcBiK4ldoXZnrEKabLXllInjVvc8p6MoBxW46ItINeABoJkxJglwA4OAR4B5xpg6wDz/a0Skgf/9hkA34A0RcRcuvlKB1ca1gUT5mU+s65yOEnAnKc6XVkt6updRjLOXXkBFpMJ273iAYiLiAYoDe4HewC9HNMcCffzPewMTjDHnjDE7gW1AZBwlUxFjgHshx0xx5thNnY5SJD61OpIgZ+nuWuF0FOWQAhd9Y8yPwIvAD8A+4JgxZjZQ2Rizzz/PPqCSf5FqQM67e2f4p11ARIaLyEoRWXnw4MG8tl/Q6CqMBPPnXIqTdHN9x2SrLee4+FAW4eo7U5eddmVu8Sx0OopySGG6d8ria73XBK4ASojIkIstksu0XH+jjTFvGWOaGWOaVaxY8YL34+PjOXTokBb+CGeM4dChQ8THB+e88pvcS4mTLD61OgZle84QPrOupZVrE1fKfqfDKAcU5uKszsBOY8xBABGZCLQB9otIVWPMPhGpChzwz58BVM+xfCK+7qDLlpiYSEZGBnn9F6AiR3x8PImJwbk94QD3QjbaV7HB1AjK9pwy0WrPHz2f0d+9kJe8tzgdRwVZYYr+D0ArESkOnAE6ASuBU8BQYJT/cYp//qnAxyLyEr7/DOoABepYjImJoWbNmoWIrtT56skPpLp28FTWHYTNjVIK6CfKs8hOob97Ea94+2PrmdtRpTB9+suBz4HVwHr/ut7CV+y7iMhWoIv/NcaYDcCnwEZgFnC/McYqVHqlAmSAeyGZxs0Uq43TUYLiM+tarpDDtHOtdzqKCrJCjb1jjHkSePJXk8/ha/XnNv9zwHOF2aZSgebBS2/3EubaTTlCKafjBMVcuylHTAL93YtYZKc6HUcFkf5fp6JeB9c6KshxJlrtnY4SNJnEMN1qxQ2ulSRw2uk4Koi06Kuo18/9DYdNAgujrMU70WpPvGTR3a3n7EcTLfoqqpXkNF1cq5hmtSYrykYaX2Nqs8OuQl/XN05HUUGkRV9Fte7u5cRJFpOiqGvnf4TJVjtauTZRlUNOh1FBokVfRbV+7m/YblclzVztdBRHTLLb4hJDH/cSp6OoINGir6JWNQ7SyrWJSVY7Iv3c/LzsMZX5zr6Gvu7F5HGBvIowWvRV1Ortb91Otts5nMRZE632XOP6kSTZ6XQUFQRa9FWUMvRzf8Nyux4Z5sLxnaLJDKsl54yHfm49oBsNtOirqNRQdlPbtZfJVlunozjuOAl8bTemp3sZLmyn46gipkVfRaWb3EvING5mWnpLB4ApVhsqyVFauTY6HUUVMS36KuoINje5l7LQTuUooX2f42CZbzfmhClGb9e3TkdRRUyLvoo6LWQLVeUw06JkcLX8OEcsX9nN6e5eQRyZTsdRRUiLvoo6vd1LOG3imGM3cTpKSJlitaGUnKaja63TUVQR0qKvokoMXnq4lzPbbsoZgnNHrnDxrd2Qg6YUN+mFWhFNi76KKh1caykjp5iiZ+1cwMLNdKs1nV1rdOTNCKZFX0WV3u5vOWwSWGwnOx0lJE2zWhMnWXR1rXQ6iioiWvRV1CjGWTq7VjPTaok3ykbUzK/Vpg4/2BWzr1ZWkUeLvooanV2rKS7nmKpn7VyEMN1uTRvXBspx3Okwqgho0VdRo5d7KT+Zsnxn6jodJaRNs1rjEVtvrhKhtOirqFCS01zrWssMqxW2fuwvapO5km32FfRyL3U6iioC+ulXUeEG10rixMs0q7XTUcKAMM1qTQvZTGUOOx1GBZgWfRUVermXsseuGLU3S7lc0+1WuMRwo3u501FUgGnRVxGvLMdp51rPNLs10XqzlMu13VRjg32VdvFEIC36KuJ1d3+HR2zt2rlM06zWNHZtI1EOOB1FBZAWfRXxermWss2+gk3mSqejhJXptu+P5E0ube1HEi36KrId30dL1yam263Qrp3Lk2EqssauTU/3MqejqADSoq8i28YpuMRo104BTbda0cC1m1qy1+koKkC06KvItmEim+zqbDfVnE4SlmZYLQHo6dLWfqTQoq8i17EM2LOc6drKL7CfKM8Kuy499SyeiKFFX0WuDZMA/P35qqCmW624xvUj18gep6OoANCiryJX+kSomspuU8XpJGFtptUSy4i29iOEFn0VmQ7vhL2roWE/p5OEvYOUYbld39+vb5yOowqpUEVfRMqIyOcisllENolIaxEpJyJzRGSr/7FsjvkfFZFtIrJFRLoWPr5SefB37dCwr7M5IsR0uzW1XD/RUHY7HUUVUmFb+q8Cs4wx9YBUYBPwCDDPGFMHmOd/jYg0AAYBDYFuwBsi4i7k9pXK3YaJUK0ZlL3K6SQRYabVHK9xaRdPBChw0ReRUkAH4F0AY0ymMeYo0BsY659tLNDH/7w3MMEYc84YsxPYBrQo6PaVytPPW+Gn9ZCkXTuBcoRSfGs31C6eCFCYln4t4CDwXxFZIyLviEgJoLIxZh+A/7GSf/5qQM7D/xn+aUoFVvpE32ODPo7GiDTT7NZUdx0kVbY7HUUVQmGKvgdoArxpjGkMnMLflZOH3K6Bz7XJICLDRWSliKw8ePBgISKqqLRhElzZGkprmyKQvrKakWncOtxymCtM0c8AMowxv3wCPsf3R2C/iFQF8D8eyDF/9RzLJwK5XtttjHnLGNPMGNOsYsWKhYioos6BTXBwk561UwSOk8BiO4Ub3csQbKfjqAIqcNE3xvwE7BGRX2442gnYCEwFhvqnDQWm+J9PBQaJSJyI1ATqAHoTThVY6RNBXNCgt9NJItJ0qxXV5BCNZZvTUVQBeQq5/AjgIxGJBXYAw/D9IflURH4D/AAMADDGbBCRT/H9YfAC9xtjrEJuX6n/McZ31s5VbaFkZafTRKQ5dlPOmRh6uZey2nuN03FUARSq6Btj0oBmubzVKY/5nwOeK8w2lcrTT+vh0DZofb/TSSLWSYqzwE6lh3s5z3hvdzqOKgC9IldFjg0TQdxQX7t2itJ0qxWV5SjNZYvTUVQBaNFXkcEY31k7ta6FEuWdThPR5tlNOGNi9UKtMKVFX0WGvavhyC49aycIThPPPLsJ3d0rwPI6HUddJi36KjKkTwRXDNTv5XSSqDDNak0FOQ67FjkdRV0mLfoq/Nm2r+jX7gzFyjidJiossFM5YYpB+hdOR1GXSYu+Cn97lsGJvZB0s9NJosY5YpltN4VN08B7zuk46jJo0VfhL/0L8BSDut2dThJVplmt4ewx2D7f6SjqMmjRV+HN8sKGyXBNV4hLcDpNVFliJ0OxstrFE2a06KvwtmsRnP5Zu3YckIUH6t8Em7+EzNNOx1H5pEVfhbf0iRBbEup0cTpJdEq6GbJOwdbZTidR+aRFX4UvbyZsmgr1boSYYk6niU412kGJSpD+udNJVD5p0Vfha/s834FE7dpxjsvtuw/x97Ph7HGn06h80KKvwtf6z6BYObj6OqeTRLfkAWCdg83TnU6i8kGLvgpP507ClpnQsA+4Y5xOE90Sm0GZq2C9dvGEAy36KjxtmQlZpyGpv9NJlAgk94cdC+Ck3t401GnRV+Ep/XMoVc13L1zlvKT+YCzYONnpJOoStOir8HP6MGybC0n9wKUf4ZBQuQFUaug7zqJCmv7GqPCzcQrYXt8BRBU6km+GPcvhyG6nk6iL0KKvws/6z6F8HaiS4nQSldMvp87qsAwhTYu+Ci/HfoTdS3wHDkWcTqNyKlsDqrfUs3hCnBZ9FV7SPweMdu2EquQBcGAD/JTudBKVBy36Krys+xQSm0P5q51OonLTsB+4PLD+U6eTqDxo0Vfh46d02J8OKQOdTqLyUqI81O4C6z7z3dFMhRwt+ip8rPvE14ps2NfpJOpiUm7x3cls12Knk6hcaNFX4cG2fAcIa3eGEhWcTqMupm5333DX67SLJxRp0VfhYdc3vtZjyi1OJ1GXElMMGvT2XU+RdcbpNOpXtOir8LDuU1/r8Rq9D25YSLkFMk/4xkhSIUWLvgp9WWd8rcYGN0FscafTqPyo0Q5KXuE7DqNCihZ9Ffo2z/C1GrVrJ3y43L4L6LbN1ZE3Q4wWfRX60j6GUolQo4PTSdTlSB3sGyNJB2ELKVr0VWg7vhd2fA2pg3REzXBTuQFUbQRrP3Y6icpBf4tUaFv3CRgbGt3qdBJVEI1ug5/W+75USNCir0KXMZA23jeIlw67EJ6S+4MrxvdzVCGh0EVfRNwiskZEpvtflxOROSKy1f9YNse8j4rINhHZIiJdC7ttFeH2roaft2grP5wVLwd1u/nG4rGynE6jCExLfySwKcfrR4B5xpg6wDz/a0SkATAIaAh0A94QEXcAtq8iVdrH4InXYRfCXeqtcOqg70we5bhCFX0RSQRuBN7JMbk3MNb/fCzQJ8f0CcaYc8aYncA2oEVhtq8imPecb9iFej0hvrTTaVRh1OkCxSv4/ogrxxW2pf8K8DCQczi9ysaYfQD+x0r+6dWAPTnmy/BPu4CIDBeRlSKy8uBBPcc3Km35Es4ehUaDnU6iCssd4xsZdctMOPWz02miXoGLvoj0BA4YY1bld5FcppncZjTGvGWMaWaMaVaxYsWCRlThbPUHULo61LrO6SQqEBoPATtLr9ANAYVp6bcFbhKRXcAE4HoRGQfsF5GqAP7HA/75M4DqOZZPBPYWYvsqUh3ZDdu/9hUKlx72iQiVG/hufrP6A99ZWcoxBS76xphHjTGJxpga+A7QzjfGDAGmAkP9sw0FpvifTwUGiUiciNQE6gArCpxcRa60j3yPjW5zNocKrCZ3wMHNkPGd00miWlGcpz8K6CIiW4Eu/tcYYzYAnwIbgVnA/cYYqwi2r8KZbcGaj+Dq66FM9UvPr8JHw74QU8LX2leOCUjRN8YsMMb09D8/ZIzpZIyp4388nGO+54wxVxtj6hpjdMxVdaHtX8PxDF+rUEWWuJKQ1A/SJ8K5E06niVp6Ra4KLavHQvHyULeH00lUUWgyFLJO+Qq/coQWfRU6Th70naqZOhg8sU6nUUUhsRlUrO/7464coUVfhY60cb6heLVrJ3KJQNOh8OMqHYTNIVr0VWiwLVj5X6jRHirWdTqNKkqpg8BTDL571+kkUcnjdAClANg2D47uhs5POZ1EXYYaj8wo0HL/8LTgxpUf03JJW04SvFtg7hp1Y9C2Faq0pa9Cw8p3oUQl31g7KuKNszpTQs7R1/2N01Gijrb0VYEUtIWXm2ocZHHcV/zb6s2//m9OwNarQtc6czXr7JoMcc/lQ6sLuY/SooqCtvSV4wZ75mOA8d7rnY6igmic1Zm6rgyayxano0QVLfrKUTF4Gej+mvl2Y/ZSwek4KoimWa05boozxKPj7AeTFn3lqO6uFVSU43xkdXY6igqyM8TzhdWe7q7lVOSo03GihhZ95ahhnlnssKuw0E5xOopywAfWDcSKxW3a2g8aLfrKMY1lK41d23jf6orRj2JU2mmqMs9qzG3uucSi99ANBv1NU44Z5pnFcVOML6wOTkdRDvqv1Y2KcpxerqVOR4kKWvSVI6pwiB6u5XxiXccpijkdRznoGzuJ7+1q3OWZSR4301MBpEVfOeJ2zxwEw1jrBqejKMcJ71ndaejaTQvZ7HSYiKdFXwVdPOe41T2fOXYzMkwlp+OoEDDZassRk8BdnllOR4l4WvRV0PVzf0NZOcl/vd2cjqJCxFni+Ni6ni6ulVwp+52OE9G06KugcmEz3D2dNLsWy009p+OoEPK+tyte3PzWHbghPtSFtOiroOruWkEN137GeG9Cx1tROR2kLF9Y7bnFvZAKHHM6TsTSoq+CyHCvZyo77CrMtps5HUaFoLetnsTg5U7t2y8yWvRV0LR1pZPs2sV/rF7Y+tFTudhpqjLLbs7t7jmU4IzTcSKS/uapoLnXPY39pgyTrHZOR1EhbIy3F6XlNIPd852OEpG06KugSJIdtHen8563O5nEOB1HhbB15mqWWA252/OlDs1QBLToq6AY6ZnEMVOcj61OTkdRYeAN6yaqyBFucS9wOkrE0aKvilxD2UkX9yre8fbgRBDvh6rC1xI7ie/sa7jPM0Vb+wGmRV8VuQc9X3DMFOd9Sy/GUvklvOK9mSvksLb2A0yLvipSSbKDLu7VvO29UVv56rIssZNYYdflfm3tB5QWfVWkHvR8wVFTgrFWV6ejqLDja+1XlcMMdH/tdJiIoUVfFZlk2UFn9xrty1cF9q3dkBV2Xe7zTCWOTKfjRAQt+qrIPOyZwBGTwPvaylcFJrzs7U9VOczt7jlOh4kIWvRVkWjvWkd7dzqveftyUlv5qhCW2g1ZYKXye89kSnHS6ThhT4u+CjjB5lHPePbYFRlndXY6jooAo7yDKcVp7vNMdTpK2Ctw0ReR6iLytYhsEpENIjLSP72ciMwRka3+x7I5lnlURLaJyBYR0f/5I1Qf1xIauHbzT+8tevWtCojN5kom2e0Y5v6KK/jZ6ThhrTAtfS/wR2NMfaAVcL+INAAeAeYZY+oA8/yv8b83CGgIdAPeEBF3YcKr0BNHJn+M+Yz1dg2m2a2djqMiyL+yBgDwh5jPHU4S3gpc9I0x+4wxq/3PTwCbgGpAb2Csf7axQB//897ABGPMOWPMTmAb0KKg21ehaaj7KxLlZ5733orR3kMVQHupwPvWDfRzLaaB7HI6TtgKyG+liNQAGgPLgcrGmH3g+8MA/HIT1GrAnhyLZfin5ba+4SKyUkRWHjx4MBARVRBU4ggPeCYx12rMt3aS03FUBPq3tzdHSOBvMe8Dxuk4YanQRV9EEoAvgAeNMccvNmsu03L9qRlj3jLGNDPGNKtYsWJhI6ogeTTmY2Lw8rT3DqejqAh1nARe8A6iuet7+riWOB0nLBWq6ItIDL6C/5ExZqJ/8n4Rqep/vypwwD89A6ieY/FEYG9htq9CRwvZRF/3Ev5j9eQHU9npOCqCfWZdS5pdi8diPiaB007HCTuFOXtHgHeBTcaYl3K8NRUY6n8+FJiSY/ogEYkTkZpAHWBFQbevQocbi7/FjCXDVOANb2+n46gIZ3DxRNYwKnCMBzyTnI4TdgrT0m8L3A5cLyJp/q8ewCigi4hsBbr4X2OM2QB8CmwEZgH3G2OsQqVXIWGIey71XT/wbNYQzhLndBwVBdaZq/nE6sgw9yzqSIbTccKKp6ALGmO+Ifd+eoBc75RhjHkOeK6g21ShJ1EO8rBnAgutFGbZzZ2Oo6LIP70DucG9kn/EvMXNmU/pfZfzSb9LqhAMf/e8g0F4LOs35N0GUCrwDlOKv2UNpbFrG8PcM52OEza06KsC6+9eRAf3el7wDuJH9CwrFXxT7dbMtRrzJ89nXCU/OR0nLGjRVwVSkSM87vmQ5XY9HV9HOUj4a9ZvyMLNKM87CLbTgUKeFn11+YxhVMw7xJHFI1m/1StvlaP2U46/e2+jtXsjQ92znY4T8vS3VV2+FW/Ryb2Gv3tvZaep6nQapZhgXcdcqzGPesZTT35wOk5I06KvLs9P6TD7ceZajfnAusHpNEr5CQ9n3cMxSjA65jXiOed0oJClRV/lX+Zp+OI3UKwMD2fdg56to0LJYUrxh6zfcY3rR/7q+cjpOCFLi77KH2Ng5sNwcDP0HcNhSjmdSKkLfGMnM8bbk9s9c+nhWuZ0nJBU4IuzVGDVeGSG0xEu6jb3XJ6L+ZDXvb158e0zTsdRKk//8t5CM9f3/DPmP2zPvIIt5kqnI4UUbemrS2omm3nKM5b5ViNe8g5wOo5SF5WFh99ljuQkxXgr5iVK6311z6NFX11UFQ7xZuyr7DEVeTDrfr3UXYWFg5Tl3syHqCKHeS3mNVx6/n42/Q1WeUrgNO/E/otinGN41h84TgmnIymVb2tMHZ7wDqODez1PesaiN13x0T59lasYvIyJeZm6soffZv2RbSbR6UhKXbZPrOuoJXu5xzOD/aYc0NPpSI7Tlr66gGDzYswY2rk38Jes37LAbuR0JKUKbJR3MJOstjwc8wmsGed0HMdp0Ve/YnjC8yG93d8yKmsQE+0OTgdSqlAMLh7OuodFVjJMfQA2TXc6kqO06KscDE96PmCY5yve8XZnjNXL6UBKBUQWHn6X9SBc0Qg+GwqbpjkdyTER3acf6ue+hxLB5m+esdzhmcPb3h48570NveJWRZJTFIPbJ8G4m+HTodD/PWjYx+lYQactfYUbi7973uUOzxzGeHtpwVeRK740DJkIic3g87tg7QSnEwWdFv0oV4IzvBvzIoM9XzPa24dR3kFowVcRLb4UDPkCrmoDk+6Bhf/wDTMSJSK6e0ddXGUO817sP6kre3g06zeMt3K9tbFSESNnl28sdzMqxtDv6+f4ZM4S/uq9C28IlcRdo24skvWGzh6qoGoum3k9djQlOMvdWX/S0zJV1Mkkhj9k/Y49piIjPZOo4fqJEZkjOEBZp6MVKe3eiTqGe9zTGB/7LKdMPP0zn9KCr6KY8LJ3AA9k3k+y7GRG3KO0dm1wOlSR0qIfRSpxhHdjXuTRmPF8ZTfjpsxn2awjECrFVLstN2U+w1FTknExf+ePnk+JJcvpWEVCi35UMPRzLWJO3J9p60rnyayh3J81kpMUdzqYUiFjm0mkd+YzTLTaM8Izmamx/0eS7HA6VsBp0Y9wNWUf/435By/FjuF7k0i3zFGMtbqiZ+godaHTxPNn773clfknyshJJsc+waOejyjJaaejBYweyI1QpTjFCM8k7nR/xVlieTrrdt63uurQyErlw3y7CTecu4a/ej7mHs8M+rkX86J3IJ9Z14b975AW/QiTwGmGumdzt+dLSnOKT61redE7kJ8p7XQ0pcLKcRL4i3c446zOPBHzAS/EvM1d7pm85u3Ll3bLsC3+WvQjRAWOcZt7LsM8sygjp5hnNeYl7wA2mBpOR1MqrK03tRiQ+SQ3upYz0vMFr8e+xlZ7ImO8vZhut+IcsU5HvCxa9MOaoYlsZYhnLje6lhEnXuZYTRjt7cd6U8vpcEpFEGGG3YovM1vQw7WCEZ5J/Ct2DI+aj5lgXcd47/X8SEWnQ+aLFv0wdJX8RG/Xt/Rxf0Mt10+cMMX42OrEh1YXdpgrnI6nVMQyuJhht2JGZkvauDZwp/srfueeyu89U1hm12eS1Y6ZVnOOk+B01Dxp0Q8DbixSZTvXu9fQ2bWaeq492EZYZtfnzayb+NJq6RtBUCkVJMK3dhLf2klU4yB93Evo517MCzFv86znPVbY9ZhrN+FruxG7TBVC6Ww5MSE+0FCzZs3MypUrC7RsuA6tHEcmDWUXjV1bae3aSEvXZkrKGbzGxXd2PebajfnSasU+yjsdVSmVzZAiO+jq/o4urlVc4/oRgB9NeZbZDVhm1yfNrs02cwUmHweBCzv2joisMsY0+/V0bek7SLCpzBFqufZRT/ZQT36gvms39WQPMWIBsMOuwlSrDUvtBiy2kzkWwv82KhXdhHXmatZ5r+afDOJK2U8H1zpauzbQ0ZXGze7FAJwwxUi3a7LZVGezuZLv7UR2mKpB+90OetEXkW7Aq4AbeMcYMyrYGYLDUIpTlJcTVOIoleUIleQI1eRnEuVnEuUANWQ/xSQze4mDpjSb7eq8Zd9Iml2bNPtqDkb44E9KRaofTGXGWV0YZ3VBsKkl+0iV7TRybSfZtZNbXAsoIeey5z9sEthpqnJr5l+L9IygoBZ9EXED/wa6ABnAdyIy1RizMeAbs7KIJQvB4MLGhfF/2bj9Xy5sPFi4xffowSIGixi8ePASK15i8RJHlu9LMoknk+Kco7icozhnKcEZEuQsCZyhtJyiFKcpIycpw8ns1npOJ0wxfjQVyDAVWGInsdNUZYepyvd2Iof0XHqlIpLBxXZTje2mWvZ9pwWb6nKQaySDGvITtWQfFeRYkZ8CGuyWfgtgmzFmB4CITAB6A4Ev+mPa8X385oCvNqfTJo5TxHPCFOMU8RwzJfiJshyzS3CYUhwxJTlkSnGAMhwwZThgynKc4oTSQR2llDMMLn4wlfnBVA7qdoNd9KsBe3K8zgBa/nomERkODPe/PCkiWwq4vQrAzwVcNlzpPkeHaNvnaNtf5IVC7/NVuU0MdtHPrYl7welDxpi3gLcKvTGRlbkdvY5kus/RIdr2Odr2F4pun4M9eEQGUD3H60Rgb5AzKKVU1Ap20f8OqCMiNUUkFhgETA1yBqWUilpB7d4xxnhF5PfAV/hO2XzPGFOU9yYrdBdRGNJ9jg7Rts/Rtr9QRPsc8lfkKqWUCpzwHBBaKaVUgWjRV0qpKBIRRV9EuonIFhHZJiKP5PK+iMho//vrRKSJEzkDJR/7e5t/P9eJyLcikupEzkC61D7nmK+5iFgi0j+Y+YpCfvZZRDqKSJqIbBCRhcHOGGj5+GyXFpFpIrLWv8/DnMgZKCLynogcEJH0PN4PfO0yxoT1F74DwtuBWkAssBZo8Kt5egAz8V0n0ApY7nTuIt7fNkBZ//Pu4by/+d3nHPPNB74E+judOwg/5zL4rma/0v+6ktO5g7DPjwEv+J9XBA4DsU5nL8Q+dwCaAOl5vB/w2hUJLf3soR2MMZnAL0M75NQb+MD4LAPKiEjVYAcNkEvurzHmW2PMEf/LZfiuhwhn+fkZA4wAvgAOBDNcEcnPPt8KTDTG/ABgjAn3/c7PPhugpIgIkICv6HuDGzNwjDGL8O1DXgJeuyKh6Oc2tEO1AswTLi53X36Dr6UQzi65zyJSDegLjAlirqKUn5/zNUBZEVkgIqtE5I6gpSsa+dnn14H6+C7qXA+MNMbYwYnniIDXrkgYTz8/Qzvka/iHMJHvfRGR6/AV/XZFmqjo5WefXwH+YoyxfI3AsJefffYATYFOQDFgqYgsM8Z8X9Thikh+9rkrkAZcD1wNzBGRxcaY40WczSkBr12RUPTzM7RDJA3/kK99EZEU4B2guzHmUJCyFZX87HMzYIK/4FcAeoiI1xgzOSgJAy+/n+ufjTGngFMisghIBcK16Odnn4cBo4yvw3ubiOwE6gErghMx6AJeuyKheyc/QztMBe7wHwlvBRwzxuwLdtAAueT+isiVwETg9jBu9eV0yX02xtQ0xtQwxtQAPgfuC+OCD/n7XE8B2ouIR0SK4xuxdlOQcwZSfvb5B3z/2SAilYG6wI6gpgyugNeusG/pmzyGdhCRe/3vj8F3NkcPYBtwGl9rISzlc3+fAMoDb/hbvl4TxiMU5nOfI0p+9tkYs0lEZgHrABvfnehyPfUvHOTz5/wM8L6IrMfX9fEXY0zYDrksIuOBjkAFEckAngRioOhqlw7DoJRSUSQSuneUUkrlkxZ9pZSKIlr0lVIqimjRV0qpKKJFXymloogWfaXChIjcKSJXOJ1DhTct+kqFjzsBLfqqULToq7AmIjVEZLOIjPWPN/65/+pURKSTiKwRkfX+ccvj/NNHichG//wv5rLOp/zrmy0iu0Skn4j8w7+eWSISc4n17xKRv4vIUhFZKSJNROQrEdn+y4VG/vn+LCLf+XP8Lcf+bBKRt/3jxc8WkWLiuz9AM+Aj8Y2fX6zov7sqEmnRV5GgLvCWMSYFOA7cJyLxwPvAQGNMMr6rz38nIuXwjcbZ0D//s3ms82rgRnxD244Dvvav5wxwY17rz7H8HmNMa2Cxf77++MZDfxpARG4A6uAbTrgR0FREOviXrQP82xjTEDgK3GyM+RxYCdxmjGlkjDlTsG+VinZa9FUk2GOMWeJ/Pg7fqKJ1gZ05xh4ai++GFceBs8A7ItIP36XtuZlpjMnCN3yvG5jln74eqHGR9f9iao75lxtjThhjDgJnRaQMcIP/aw2wGt+gYXX8y+w0xqT5n6/yb0+pgNCiryLBr8cSMeQ+JC3GGC++1vUXQB/+V8x/7Zx/fhvIMv8br8TG16q/1PjN53LMfy7H9JzLP+9vtTcyxtQ2xrz7q2UBLCJgjCwVOrToq0hwpYi09j8fDHwDbAZqiEht//TbgYUikgCUNsZ8CTyIr2ulIHJd/2Us/xVwlz8PIlJNRCpdYpkTQMnLTqpUDtqCUJFgEzBURP4DbAXeNMacFd9Nsz8TEQ++YXvHAOWAKf4+eQEeKsgGL7L+/C4/W0Tq47vxCcBJYAi+ln1e3gfGiMgZoLX266uC0FE2VVgTkRrAdGNMktNZlAoH2r2jlFJRRFv6SikVRbSlr5RSUUSLvlJKRREt+kopFUW06CulVBTRoq+UUlHk/wEFeP3GvqyAqQAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plot sia_prop_moments histogram\n",
    "plt.hist(sia_prop_moments, bins=7)\n",
    "\n",
    "# plot gaussian curve\n",
    "c1 = np.arange(0.0, 1.0, 0.01)\n",
    "t1 = 1000* np.exp(-(c1-0.56)**2/(2*0.14**2))\n",
    "plt.plot(c1,t1)\n",
    "plt.xlabel(property + ' moment')\n",
    "plt.legend(['Histogram', 'Gaussian\\n$\\mu=0.56, \\sigma=0.14$'], loc='upper left')\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fc555766",
   "metadata": {},
   "source": [
    "The moment suggests that a positive sentiment is not particularly skewed necessarily towards the beginning or the end of the Tweet. This conclusion is somewhat preliminary however, since we used a sentiment analysis tool which is not specialized to Twitter."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "6ac078d9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD4CAYAAAAAczaOAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAg/0lEQVR4nO3de3RU9dX/8fcW0KAIAgaLhDbYFZV7BMS02Ac0FdEWUZco9QIsbBHEiq3aB9ra6nrKU1qteCtYaxW0CKQgynLVtvwC1ksRTWgEA6WARkzhAQyKeMFy2b8/5ks6hkkyAzND4Hxea82aM/uc7zn7nBk2J99z5jvm7oiISDQcc7gTEBGR7FHRFxGJEBV9EZEIUdEXEYkQFX0RkQhpfrgTaMzJJ5/s+fn5hzsNEZEjSnl5+Xvunls33uSLfn5+PmVlZYc7DRGRI4qZvZMoru4dEZEIUdEXEYkQFX0RkQhp8n36InJk2717N9XV1ezatetwp3JUysnJIS8vjxYtWiS1vIq+iGRUdXU1J554Ivn5+ZjZ4U7nqOLu1NTUUF1dTZcuXZJqo+4dEcmoXbt20b59exX8DDAz2rdvn9JfUSr6IpJxKviZk+qxVdEXEYkQ9emLSFZNW/zPtK7vexecntb1NVUzZ86krKyMhx566JDWo6IvcpC2PVj/P77c796UxUxEkqfuHRGJhCeeeIJevXrRu3dvrrvuOt555x2Ki4vp1asXxcXFbNy4EYDRo0czfvx4zjvvPE477TT++te/MmbMGLp27cro0aNr19eqVStuvfVW+vTpQ3FxMdu2bQOgoqKCoqIievXqxWWXXcb7778PwKBBg2qHlHnvvffYP6bYzJkzufzyyxkyZAgFBQX84Ac/qN3G448/zumnn87AgQN55ZVX0nIcVPRF5KhXWVnJlClTWLJkCW+88Qb3338/N910EyNHjmTlypVcc8013HzzzbXLv//++yxZsoRp06YxdOhQvve971FZWcmqVauoqKgA4OOPP6ZPnz6sWLGCgQMHctdddwEwcuRIfvGLX7By5Up69uxZG29IRUUF8+bNY9WqVcybN493332XzZs389Of/pRXXnmFxYsXs3r16rQcCxV9ETnqLVmyhCuuuIKTTz4ZgHbt2rFs2TKuvvpqAK677jpefvnl2uWHDh2KmdGzZ09OOeUUevbsyTHHHEP37t2pqqoC4JhjjuGqq64C4Nprr+Xll19mx44dfPDBBwwcOBCAUaNG8eKLLzaaX3FxMW3atCEnJ4du3brxzjvvsHz5cgYNGkRubi7HHnts7bYOlYq+iBz13L3RWxvj5x933HFArLDvn97/es+ePY22T6R58+bs27cP4ID76uO30axZs9ptZOJWVxV9ETnqFRcXU1JSQk1NDQDbt2/nq1/9KnPnzgVg9uzZnHvuuSmtc9++fcyfPx+Ap556inPPPZc2bdrQtm1bXnrpJQCefPLJ2rP+/Px8ysvLAWrbNeScc87hhRdeoKamht27d/OHP/whpfzqo7t3RCSrDsctlt27d+dHP/oRAwcOpFmzZpx11lk88MADjBkzhrvvvpvc3Fwef/zxlNZ5wgknUFlZSd++fWnTpg3z5s0DYNasWYwbN45PPvmE0047rXa9t912G1deeSVPPvkk559/fqPr79ixI3feeSdf+cpX6NixI3369GHv3r2p73wd5u6HvJJM6tevn+tHVKQp0i2byVmzZg1du3Y93GmkXatWrfjoo48OdxpA4mNsZuXu3q/usureERGJEBV9EZGD0FTO8lOloi8iEiEq+iIiEdJo0TezM8ysIu7xoZndYmbtzGyxma0Lz23j2kw2s/VmttbMLoyL9zWzVWHeA6bxVkVEsqrRou/ua9290N0Lgb7AJ8BCYBJQ6u4FQGl4jZl1A0YA3YEhwHQzaxZWNwMYCxSEx5C07o2IiDQo1fv0i4EN7v6OmQ0DBoX4LOAF4L+BYcBcd/8MeNvM1gP9zawKaO3uywDM7AngUuD5Q9wHETmSLP15etd33uT0rq8eDz/8MMcffzwjR45k5syZDB48mFNPPRWAb3/723z/+9+nW7duWcnlUKRa9EcAc8L0Ke6+GcDdN5tZhxDvBLwa16Y6xHaH6brxA5jZWGJ/EfDFL34xxRRFRNJv3LhxtdMzZ86kR48etUX/0UcfPVxppSzpC7lmdixwCdDYd4ET9dN7A/EDg+6PuHs/d++Xm5ubbIoiIglVVVVx5plnMmrUKHr16sUVV1zBJ598QmlpKWeddRY9e/ZkzJgxfPbZZwBMmjSJbt260atXL2677TYA7rzzTu655x7mz59PWVkZ11xzDYWFhXz66ae1wybPmDHjc0Mjz5w5k+9+97sA/P73v6d///4UFhZyww03pOXbtQcjlbt3LgJWuPuW8HqLmXUECM9bQ7wa6BzXLg/YFOJ5CeIiIhm3du1axo4dy8qVK2ndujX33nsvo0ePrh3SeM+ePcyYMYPt27ezcOFCKisrWblyJT/+8Y8/t54rrriCfv36MXv2bCoqKmjZsuXn5j399NO1r+fNm8dVV13FmjVrmDdvHq+88goVFRU0a9aM2bNnZ23f46VS9L/Ff7p2ABYBo8L0KODZuPgIMzvOzLoQu2D7WugK2mlmReGunZFxbUREMqpz584MGDAAiA2FXFpaSpcuXTj99NhYQPuHQW7dujU5OTl8+9vf5umnn+b4449Pehu5ubmcdtppvPrqq9TU1LB27VoGDBhAaWkp5eXlnH322RQWFlJaWspbb72Vkf1sTFJ9+mZ2PHABcENceCpQYmbXAxuB4QDuXmlmJcBqYA8wwd33/x0zHpgJtCR2AVcXcUUkK5K9Q7x58+a89tprlJaWMnfuXB566CGWLFmS9HauuuoqSkpKOPPMM7nsssswM9ydUaNG8fOfp/ki9kFI6kzf3T9x9/buviMuVuPuxe5eEJ63x82b4u5fdvcz3P35uHiZu/cI827ypj7am4gcNTZu3MiyZcsAmDNnDl//+tepqqpi/fr1wH+GQf7oo4/YsWMHF198Mffdd1/tL2XFO/HEE9m5c2fC7Vx++eU888wzzJkzp/aHT4qLi5k/fz5bt8Z6wbdv384777yTgb1snIZWFpHsytItlnV17dqVWbNmccMNN1BQUMD9999PUVERw4cPZ8+ePZx99tmMGzeO7du3M2zYMHbt2oW7M23atAPWNXr0aMaNG0fLli1r/yPZr23btnTr1o3Vq1fTv39/ALp168bPfvYzBg8ezL59+2jRogW//vWv+dKXvpSVfY+noZVFDpKGVk5OUxhauaqqim9+85u8+eabhzWPTNHQyiIikpCKvogc9fLz84/as/xUqeiLiESIir6ISISo6IuIRIiKvohIhOg+fRHJqukV09O6vhsLb0zr+hpSVVXF3/72N66++uqU27Zq1apJ/K6uzvRFRJJUVVXFU089lXDenj17spzNwVHRF5GjXlVVFV27duU73/kO3bt3Z/DgwXz66ads2LCBIUOG0LdvX772ta/xj3/8A4h943b+/Pm17Vu1agXEhlx+6aWXKCwsZNq0acycOZPhw4czdOhQBg8ezEcffURxcTF9+vShZ8+ePPts0xtTUt07IhIJ69atY86cOfz2t7/lyiuvZMGCBTz++OM8/PDDFBQUsHz5cm688cYGB1ebOnUq99xzD8899xwQGy9/2bJlrFy5knbt2rFnzx4WLlxI69atee+99ygqKuKSSy5JerC3bFDRF5FI6NKlC4WFhQD07du3tn9++PDhtcvs/xGVVFxwwQW0a9cOAHfnhz/8IS+++CLHHHMM//rXv9iyZQtf+MIX0rIP6aCiLyKRcNxxx9VON2vWjC1btnDSSSclHEWzefPm7Nu3D4gV8n//+9/1rveEE06onZ49ezbbtm2jvLycFi1akJ+fz65du9K3E2mgPn0RiaTWrVvTpUsX/vCH2C/AujtvvPEGEBu2oby8HIBnn32W3bt3Aw0PqQywY8cOOnToQIsWLVi6dOlhGz65ITrTF5GsyuYtlo2ZPXs248eP52c/+xm7d+9mxIgR9O7dm+985zsMGzaM/v37U1xcXHs236tXL5o3b07v3r0ZPXo0bdu2/dz6rrnmGoYOHUq/fv0oLCzkzDPPPBy71SANrSxykDS0cnKawtDKRzsNrSwiIgmp6IuIREhSRd/MTjKz+Wb2DzNbY2ZfMbN2ZrbYzNaF57Zxy082s/VmttbMLoyL9zWzVWHeA9aUbl4VEYmAZM/07wf+5O5nAr2BNcAkoNTdC4DS8Boz6waMALoDQ4DpZtYsrGcGMBYoCI8hadoPERFJQqNF38xaA/8F/A7A3f/t7h8Aw4BZYbFZwKVhehgw190/c/e3gfVAfzPrCLR292Ueu3r8RFwbERHJgmTO9E8DtgGPm9nfzexRMzsBOMXdNwOE5w5h+U7Au3Htq0OsU5iuGz+AmY01szIzK9u2bVtKOyQiIvVL5j795kAf4LvuvtzM7id05dQjUT+9NxA/MOj+CPAIxG7ZTCJHETlCNHSr68FoCrfHfvDBBzz11FPceGPsOwibNm3i5ptv/tygbU1FMmf61UC1uy8Pr+cT+09gS+iyITxvjVu+c1z7PGBTiOcliIuIHNE++OADpk//z+8EnHrqqU2y4EMSRd/d/w9418zOCKFiYDWwCBgVYqOA/WOILgJGmNlxZtaF2AXb10IX0E4zKwp37YyMayMikjGpDq28YcMGioqKOPvss/nJT35SO7RyfUMnT5o0iQ0bNlBYWMjtt99OVVUVPXr0AOCcc86hsrKyNpdBgwZRXl7Oxx9/zJgxYzj77LM566yzsjYMc7J373wXmG1mK4FC4H+BqcAFZrYOuCC8xt0rgRJi/zH8CZjg7nvDesYDjxK7uLsBeD49uyEi0rB169YxYcIEKisrOemkk1iwYAFjx47lwQcfpLy8nHvuuae2e2bixIlMnDiR119/nVNPPbV2HTk5OSxcuJAVK1awdOlSbr31VtydqVOn8uUvf5mKigruvvvuz213xIgRlJSUALB582Y2bdpE3759mTJlCueffz6vv/46S5cu5fbbb+fjjz/O+HFIauwdd68ADvg6L7Gz/kTLTwGmJIiXAT1SyE9EJC1SGVp52bJlPPPMMwBcffXV3HbbbUD9Qyc35Morr+SCCy7grrvuoqSkpHZ7f/nLX1i0aBH33HMPALt27WLjxo0ZH7JCA66JSCSkMrRyfQ5m6OROnTrRvn17Vq5cybx58/jNb34DxP4DWbBgAWeccUaD7dNNwzCISCQ1NLRyUVERCxYsAGDu3Lm1beobOrmxIZdHjBjBL3/5S3bs2EHPnj0BuPDCC3nwwQfZP+jl3//+9/TvZAI60xeRrGoKt1juV9/Qyvfddx/XXnstv/rVr/jGN75BmzZtgPqHTm7fvj0DBgygR48eXHTRRUyYMOFz27niiiuYOHEid9xxR23sjjvu4JZbbqFXr164O/n5+bU/w5hJGlpZ5CBpaOXkHIlDK3/yySe0bNkSM2Pu3LnMmTOnSf7I+X6pDK2sM30RkTrKy8u56aabcHdOOukkHnvsscOdUtqo6IuI1PG1r32ttn//aKMLuSKScU29G/lIluqxVdEXkYzKycmhpqZGhT8D3J2amhpycnKSbqPuHRHJqLy8PKqrq9GIuZmRk5NDXl5e4wsGKvoiklEtWrSgS5cuhzsNCdS9IyISISr6IiIRoqIvIhIhKvoiIhGioi8iEiEq+iIiEaKiLyISISr6IiIRklTRN7MqM1tlZhVmVhZi7cxssZmtC89t45afbGbrzWytmV0YF+8b1rPezB4IP5AuIiJZksqZ/nnuXhg3PvMkoNTdC4DS8Boz6waMALoDQ4DpZtYstJkBjAUKwmPIoe+CiIgk61C6d4YBs8L0LODSuPhcd//M3d8G1gP9zawj0Nrdl3ls5KUn4tqIiEgWJFv0HfiLmZWb2dgQO8XdNwOE5w4h3gl4N65tdYh1CtN14wcws7FmVmZmZRqkSUQkfZIdcG2Au28ysw7AYjP7RwPLJuqn9wbiBwbdHwEegdjPJSaZo4iINCKpM3133xSetwILgf7AltBlQ3jeGhavBjrHNc8DNoV4XoK4iIhkSaNF38xOMLMT908Dg4E3gUXAqLDYKGD/rwYvAkaY2XFm1oXYBdvXQhfQTjMrCnftjIxrIyIiWZBM984pwMJwd2Vz4Cl3/5OZvQ6UmNn1wEZgOIC7V5pZCbAa2ANMcPe9YV3jgZlAS+D58BARkSxptOi7+1tA7wTxGqC4njZTgCkJ4mVAj9TTFBGRdNA3ckVEIkRFX0QkQlT0RUQiREVfRCRCVPRFRCJERV9EJEJU9EVEIkRFX0QkQlT0RUQiREVfRCRCVPRFRCJERV9EJEJU9EVEIkRFX0QkQlT0RUQiREVfRCRCVPRFRCJERV9EJEKSLvpm1szM/m5mz4XX7cxssZmtC89t45adbGbrzWytmV0YF+9rZqvCvAfCD6SLiEiWpHKmPxFYE/d6ElDq7gVAaXiNmXUDRgDdgSHAdDNrFtrMAMYCBeEx5JCyFxGRlCRV9M0sD/gG8GhceBgwK0zPAi6Ni89198/c/W1gPdDfzDoCrd19mbs78ERcGxERyYJkz/TvA34A7IuLneLumwHCc4cQ7wS8G7dcdYh1CtN14wcws7FmVmZmZdu2bUsyRRERaUyjRd/MvglsdffyJNeZqJ/eG4gfGHR/xN37uXu/3NzcJDcrIiKNaZ7EMgOAS8zsYiAHaG1mvwe2mFlHd98cum62huWrgc5x7fOATSGelyAuIiJZ0mjRd/fJwGQAMxsE3Obu15rZ3cAoYGp4fjY0WQQ8ZWb3AqcSu2D7mrvvNbOdZlYELAdGAg+md3dEsqjqpfrnLd35n+nzJmc+F5EkJXOmX5+pQImZXQ9sBIYDuHulmZUAq4E9wAR33xvajAdmAi2B58NDRESyJKWi7+4vAC+E6RqguJ7lpgBTEsTLgB6pJikiIumhb+SKiESIir6ISISo6IuIRIiKvohIhKjoi4hEiIq+iEiEqOiLiESIir6ISISo6IuIRIiKvohIhKjoi4hEiIq+iEiEqOiLiESIir6ISISo6IuIRIiKvohIhKjoi4hEiIq+iEiEqOiLiERIo7+Ra2Y5wIvAcWH5+e7+UzNrB8wD8oEq4Ep3fz+0mQxcD+wFbnb3P4d4X/7zw+h/BCa6u6d3l0SamKU/T2658yZnNg8RkjvT/ww43917A4XAEDMrAiYBpe5eAJSG15hZN2AE0B0YAkw3s2ZhXTOAsUBBeAxJ366IiEhjGi36HvNReNkiPBwYBswK8VnApWF6GDDX3T9z97eB9UB/M+sItHb3ZeHs/om4NiIikgVJ9embWTMzqwC2AovdfTlwirtvBgjPHcLinYB345pXh1inMF03nmh7Y82szMzKtm3blsLuiIhIQ5Iq+u6+190LgTxiZ+09GljcEq2igXii7T3i7v3cvV9ubm4yKYqISBJSunvH3T8AXiDWF78ldNkQnreGxaqBznHN8oBNIZ6XIC4iIlmSzN07ucBud//AzFoCXwd+ASwCRgFTw/Ozocki4Ckzuxc4ldgF29fcfa+Z7QwXgZcDI4EH071DIiJN2fSK6Uktd2PhjRnZfqNFH+gIzAp34BwDlLj7c2a2DCgxs+uBjcBwAHevNLMSYDWwB5jg7nvDusbzn1s2nw8PERHJkkaLvruvBM5KEK8BiutpMwWYkiBeBjR0PUBERDIomTN9EZHImLb4nxld/4oPa5Ja7sbCzGxfwzCIiESIzvRFRNJkxYfzDncKjdKZvohIhKjoi4hEiIq+iEiEqOiLiESIir6ISISo6IuIRIhu2ZRISscXcAa9/2m989a/ldwXcL5yWvtDzkMy70i4FTNZOtMXEYkQFX0RkQhR0RcRiRAVfRGRCFHRFxGJEBV9EZEIUdEXEYkQFX0RkQhptOibWWczW2pma8ys0swmhng7M1tsZuvCc9u4NpPNbL2ZrTWzC+Pifc1sVZj3gJlZZnZLREQSSeZMfw9wq7t3BYqACWbWDZgElLp7AVAaXhPmjQC6A0OA6eFH1QFmAGOBgvAYksZ9ERGRRjRa9N19s7uvCNM7gTVAJ2AYMCssNgu4NEwPA+a6+2fu/jawHuhvZh2B1u6+zN0deCKujYiIZEFKffpmlg+cBSwHTnH3zRD7jwHoEBbrBLwb16w6xDqF6brxRNsZa2ZlZla2bdu2VFIUEZEGJF30zawVsAC4xd0/bGjRBDFvIH5g0P0Rd+/n7v1yc3OTTVFERBqRVNE3sxbECv5sd386hLeELhvC89YQrwY6xzXPAzaFeF6CuIiIZEkyd+8Y8DtgjbvfGzdrETAqTI8Cno2LjzCz48ysC7ELtq+FLqCdZlYU1jkyro2IiGRBMuPpDwCuA1aZWUWI/RCYCpSY2fXARmA4gLtXmlkJsJrYnT8T3H1vaDcemAm0BJ4PDxERyZJGi767v0zi/niA4nraTAGmJIiXAT1SSVBERNJH38gVEYkQ/VyiSFOx9OeNL3Pe5MznIUc1nemLiESIir6ISISo6IuIRIiKvohIhKjoi4hEiIq+iEiEqOiLiESIir6ISIToy1ki0mRMW/zPw53CUU9n+iIiEaIzfZEMyHml/jPWXQNOz2ImydEZdnToTF9EJEJU9EVEIkRFX0QkQlT0RUQiREVfRCRCkvlh9MfMbKuZvRkXa2dmi81sXXhuGzdvspmtN7O1ZnZhXLyvma0K8x4IP44uIiJZlMyZ/kxgSJ3YJKDU3QuA0vAaM+sGjAC6hzbTzaxZaDMDGAsUhEfddYqISIYl88PoL5pZfp3wMGBQmJ4FvAD8d4jPdffPgLfNbD3Q38yqgNbuvgzAzJ4ALgWeP+Q9EBE5SCs+nHe4U8i6g+3TP8XdNwOE5w4h3gl4N2656hDrFKbrxhMys7FmVmZmZdu2bTvIFEVEpK50X8hN1E/vDcQTcvdH3L2fu/fLzc1NW3IiIlF3sEV/i5l1BAjPW0O8Gugct1wesCnE8xLERUQkiw626C8CRoXpUcCzcfERZnacmXUhdsH2tdAFtNPMisJdOyPj2oiISJY0eiHXzOYQu2h7splVAz8FpgIlZnY9sBEYDuDulWZWAqwG9gAT3H1vWNV4YncCtSR2AVcXcUVEsiyZu3e+Vc+s4nqWnwJMSRAvA3qklJ2IiKSVvpErIhIhKvoiIhGioi8iEiEq+iIiEaKfSxQ5jJa9VZPS8q/u0c8ayqFR0Rc5ghRtfCSp5V794tgMZyJHKnXviIhEiIq+iEiEqOiLiESIir6ISIToQq5IluW8Uv8dOLsGnJ7FTCSKVPQjZNriw3+73/cuUFGTzIviL2IlS907IiIRojN9yaqm8NeGSJTpTF9EJEJU9EVEIkTdO1mgLg3JNg3XIPVR0RdpQnQ7p2TaUV30dYYtcnTRrZiHLut9+mY2xMzWmtl6M5uU7e2LiERZVs/0zawZ8GvgAqAaeN3MFrn76mzmIXIkykTXj/r+oyfb3Tv9gfXu/haAmc0FhgEq+iKHINPXArLxn4O6brIj20W/E/Bu3Otq4Jy6C5nZWGD/p+cjM1t7kNs7GXjvINtmkvJKjfJKzefzejybm/5VQzOPjOPVRPyenxxqXl9KFMx20bcEMT8g4P4IkNypRUMbMytz936Hup50U16pUV6pUV6piVpe2b6QWw10jnudB2zKcg4iIpGV7aL/OlBgZl3M7FhgBLAoyzmIiERWVrt33H2Pmd0E/BloBjzm7pUZ3OQhdxFliPJKjfJKjfJKTaTyMvcDutRFROQopQHXREQiREVfRCRCjuiib2bDzazSzPaZWb23NtU39IOZtTOzxWa2Ljy3TVNeja7XzM4ws4q4x4dmdkuYd6eZ/Stu3sXZyissV2Vmq8K2y1Jtn6nczKyzmS01szXhfZ8YNy9tx6yxoUIs5oEwf6WZ9Um27aFIIq9rQj4rzexvZtY7bl7C9zSLuQ0ysx1x789Pkm2b4bxuj8vpTTPba2btwryMHDMze8zMtprZm/XMz+zny92P2AfQFTgDeAHoV88yzYANwGnAscAbQLcw75fApDA9CfhFmvJKab0hx/8DvhRe3wncloHjlVReQBVw8qHuV7pzAzoCfcL0icA/497LtByzhj4vcctcDDxP7HsnRcDyZNtmOK+vAm3D9EX782roPc1iboOA5w6mbSbzqrP8UGBJpo8Z8F9AH+DNeuZn9PN1RJ/pu/sad2/s27q1Qz+4+7+B/UM/EJ5nhelZwKVpSi3V9RYDG9z9nTRtvz6Hur+ZOl5JrdvdN7v7ijC9E1hD7Fve6dTQ5yU+1yc85lXgJDPrmGTbjOXl7n9z9/fDy1eJfQ8mGw5lvw/rMavjW8CcNG27Xu7+IrC9gUUy+vk6oot+khIN/bC/UJzi7pshVlCADmnaZqrrHcGBH7abwp92j6WxGyXZvBz4i5mVW2xIjFTbZzI3AMwsHzgLWB4XTscxa+jz0tgyybQ9WKmu+3piZ4v71feeZjO3r5jZG2b2vJl1T7FtJvPCzI4HhgAL4sKZPGYNyejnq8mPp29m/w/4QoJZP3L3Z5NZRYLYId+n2lBeKa7nWOASYHJceAbwP8Ty/B9iA5qMyWJeA9x9k5l1ABab2T/C2ckhSeMxa0XsH+ct7v5hCB/0Mau7+gSxup+X+pbJyGetkW0euKDZecSK/rlx4Yy8pynktoJY9+VH4XrLM0BBkm0zmdd+Q4FX3D3+DDyTx6whGf18Nfmi7+5fP8RVNDT0wxYz6+jum8OfT1vTkZeZpbLei4AV7r4lbt2102b2W+C5bObl7pvC81YzW0jsz8oXOYTjla7czKwFsYI/292fjlv3QR+zOpIZKqS+ZY5Nou3BSmoIEzPrBTwKXOTuNfvjDbynWckt7j9n3P2PZjbdzE5Opm0m84pzwF/bGT5mDcno5ysK3TsNDf2wCBgVpkcByfzlkIxU1ntAP2IoevtdBiS8yp+JvMzsBDM7cf80MDhu+5k6XsnmZsDvgDXufm+deek6ZskMFbIIGBnusigCdoQuqUwOM9Lous3si8DTwHXu/s+4eEPvabZy+0J4/zCz/sRqT00ybTOZV8inDTCQuM9cFo5ZQzL7+Ur3lelsPoj9464GPgO2AH8O8VOBP8YtdzGxOz02EOsW2h9vD5QC68JzuzTllXC9CfI6ntgHv02d9k8Cq4CV4U3tmK28iN0Z8EZ4VGbjeKWQ27nE/pxdCVSEx8XpPmaJPi/AOGBcmDZiPwa0IWyzX0Nt03iMGsvrUeD9uGNT1th7msXcbgrbfoPYReavNoVjFl6PBubWaZexY0bsJG8zsJtY/bo+m58vDcMgIhIhUejeERGRQEVfRCRCVPRFRCJERV9EJEJU9EVEIkRFX0QkQlT0RUQi5P8D8P0O38jyEMwAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "compound_score = [sia_score['compound'] for sia_score in sia_scores]\n",
    "pos_score = [sia_score['pos'] for sia_score in sia_scores]\n",
    "neu_score = [sia_score['neu'] for sia_score in sia_scores]\n",
    "neg_score = [sia_score['neg'] for sia_score in sia_scores]\n",
    "# plot histogram of compound score\n",
    "plt.hist(compound_score, alpha=0.5, bins=9)\n",
    "plt.hist(pos_score, alpha=0.5,bins=9)\n",
    "plt.hist(neu_score, alpha=0.5,bins=9)\n",
    "plt.hist(neg_score, alpha=0.5,bins=9)\n",
    "plt.legend(['compound','positive','neutral','negative'])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b1c01b52",
   "metadata": {},
   "source": [
    "By the metric of the sentiment scores that we adopted, we find that our Tweets are more positive than negative (not a shocker), but generally neutral (perhaps not a shocker)."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e293c88b-72f8-4d45-8d3f-afc7aa68c7a3",
   "metadata": {},
   "source": [
    "# Exploratory ROUGE"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f4ee3a70",
   "metadata": {},
   "source": [
    "Here we don't really do ROUGE in the sense of evaluating given summary vs. generated summary (this we do after training, in evaluate_predictions.ipynb). We adopt the ROUGE metric as a comparison between an abstract and the real Tweet."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "6ae86fad-fd44-4ae1-b667-4a3b5749c4c5",
   "metadata": {},
   "outputs": [],
   "source": [
    "from datasets import load_metric\n",
    "\n",
    "rouge_score = load_metric(\"rouge\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "e0045248-4e3a-49cc-963a-4df81f33c77a",
   "metadata": {},
   "outputs": [],
   "source": [
    "score_list = []\n",
    "\n",
    "for label_example, text_example in zip(df_mid['Tweets'],df_mid['Abstract']):\n",
    "    scores = rouge_score.compute(predictions=[label_example], references=[text_example])\n",
    "    \n",
    "    score_list.append(scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "9941b381-9a26-4e1e-8b9d-1ca377b89019",
   "metadata": {},
   "outputs": [],
   "source": [
    "rouge1_mid_fmeasure_base = [score_list[i]['rouge1'].mid.fmeasure for i in range(len(score_list))]\n",
    "rouge2_mid_fmeasure_base = [score_list[i]['rouge2'].mid.fmeasure for i in range(len(score_list))]\n",
    "rougeL_mid_fmeasure_base = [score_list[i]['rougeL'].mid.fmeasure for i in range(len(score_list))]\n",
    "\n",
    "rouge1_mid_precision_base = [score_list[i]['rouge1'].mid.precision for i in range(len(score_list))]\n",
    "rouge2_mid_precision_base = [score_list[i]['rouge2'].mid.precision for i in range(len(score_list))]\n",
    "rougeL_mid_precision_base = [score_list[i]['rougeL'].mid.precision for i in range(len(score_list))]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "574fd3f8-4740-4ede-8dff-a4f0b673723d",
   "metadata": {},
   "source": [
    "### Baseline ROUGE score histogram"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "00e6827a-5deb-4e6f-afef-4eb7e2916d02",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYQAAAEGCAYAAABlxeIAAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAXrUlEQVR4nO3de7RdZX3u8e/DRaOSxBrAYYk2wYCCRSNEaBXxboEaVPCUW48DRTmKWArDUfVYz9HTDqr1ggOF2lQpao+I9VgVRBBRBBGEcA8gFBHNRh2EWHeKiID8zh9zxix3s7PXzl6XrJ3vZ4w92Guuueb7Wy8761nvvLwzVYUkSdsMuwBJ0pbBQJAkAQaCJKllIEiSAANBktTabtgFzMSOO+5YixYtGnYZkjRSrrnmmnuraqeJy0c6EBYtWsTKlSuHXYYkjZQkP9rYcncZSZKAEQ2EJMuTrBgfHx92KZI0a4xkIFTVuVV13Pz584ddiiTNGiN9DEGShuGhhx5ibGyMBx54YNilbNKcOXNYuHAh22+/fVfrGwiSNE1jY2PMnTuXRYsWkWTY5WxUVbF27VrGxsZYvHhxV68ZyV1GkjRMDzzwAAsWLNhiwwAgCQsWLJjWKMZAkKTNsCWHwXrTrdFAkCQBW/MxhG/93XDafdE7h9OupL459aLbe7q9k162e1frvf71r+e8885j5513ZtWqVTNu1xGCJI2oY445hgsuuKBn2xvJQPDCNEmCAw44gCc84Qk9295IBoIXpklS7229xxC2AGdcf8ZQ2j1+6fFDaVfSlm0kRwiSpN4zECRJgLuMJGnGuj1NtNeOPPJILrnkEu69914WLlzIe9/7Xo499tjN3p6BIEkj6uyzz+7p9txlJEkCDARJUstAkCQBBoIkqWUgSJIAA0GS1BrJ006TLAeWL1myZNilSFLvp9PvYpr81atX89rXvpaf/exnbLPNNhx33HGceOKJM2p2JEcITm4naWu33Xbb8aEPfYhbb72VK6+8ktNPP51bbrllRtscyUCQpK3dk570JPbee28A5s6dyx577MHdd989o20aCJI04u666y6uu+469ttvvxltx0CQpBF23333cdhhh/GRj3yEefPmzWhbBoIkjaiHHnqIww47jKOPPppDDz10xtszECRpBFUVxx57LHvssQcnn3xyT7Y5kqedStIWpYvTRHvt8ssv5zOf+Qx77bUXS5cuBeCUU07h4IMP3uxtGgiSNIL2339/qqqn23SXkSQJMBAkSS0DQZIEGAiSpJYHlbdCp150+1DaHdaNyCV1x0AYtI5ZEVff9b2BNTs2b5/f/r73zC5mlDRLGQiSNENnXH9GT7d3/NLjp1xnhx124L777utpux5DkCQBBoIkqbXF7DJKsgdwIrAjcHFV/cOQS1KPeTBb2rL1dYSQ5Mwk9yRZNWH5gUluS3JHkncAVNWtVfUm4M+AZf2sS5L0X/V7l9FZwIGdC5JsC5wOHATsCRyZZM/2uUOA7wAX97kuSdIEfQ2EqroU+PmExfsCd1TVnVX1IPA54JXt+l+pqucCR0+2zSTHJVmZZOWaNWv6VbokbXWGcQxhF2B1x+MxYL8kLwQOBR4NnD/Zi6tqBbACYNmyZb2d6k+SNkM3p4n22v3338/ChQt/+/jkk0+e8X0RhhEI2ciyqqpLgEsGW8rW6dp15wyl3b3nHT6UdqXZ6JFHHun5Nodx2ukY8OSOxwuBn0xnA0mWJ1kxPj7e08IkaWs2jBHC1cBuSRYDdwNHAEdNZwNVdS5w7rJly94402LO+MWNM92EJM0K/T7t9GzgCuBpScaSHFtVDwMnABcCtwKfr6qb+1mHJPVar+9W1g/TrbGvI4SqOnKS5eeziQPHUi95QZx6bc6cOaxdu5YFCxaQbOyw6PBVFWvXrmXOnDldv2aLuVJ5OpIsB5YvWbJk2KVI2gotXLiQsbExtvRT3+fMmfM7ZyJNZSQDoZfHECRpurbffnsWL1487DJ6zsntJEmAgSBJao1kIHgdgiT13kgGQlWdW1XHzZ8/f9ilSNKsMZKBIEnqPQNBkgSMaCB4DEGSem8kA8FjCJLUeyMZCJKk3jMQJEmAgSBJahkIkiRgRAPBs4wkqfdGMhA8y0iSem8kA0GS1HsGgiQJMBAkSS0DQZIEGAiSpNZIBoKnnUpS741kIHjaqST13kgGgiSp9wwESRJgIEiSWgaCJAkwECRJLQNBkgQYCJKk1nbDLmBzJFkOLF+yZMmwSxkZC9ddM5R2x+btM5R2JU3fSI4QvDBNknpvJANBktR7I7nLSBoFp150+9DaPulluw+tbY0uRwiSJMBAkCS1DARJEmAgSJJaBoIkCTAQJEktA0GSBHQRCEkel2Sb9vfdkxySZPv+lyZJGqRuRgiXAnOS7AJcDLwOOKufRU0lyfIkK8bHx4dZhiTNKt0EQqrqfuBQ4KNV9Wpgz/6WtWnOZSRJvddVICT5Y+Bo4KvtMqe8kKRZppsP9hOBdwL/VlU3J9kV+FZ/y9JsdO26c4bS7t7zDh9Ku9Ko6SYQnlhVh6x/UFV3JrmsjzVJkoagm11G7+xymSRphE06QkhyEHAwsEuS0zqemgc83O/CJEmDtaldRj8BVgKHAJ33X/xP4KR+FiVJGrxJA6GqbgBuSPLZdr2nVNVtA6tMkjRQ3RxDOBC4HrgAIMnSJF/pZ1GSpMHrJhDeA+wL/AKgqq4HFvWrIEnScHQTCA9XlXNESNIs1811CKuSHAVsm2Q34C+A7/a3LEnSoHUzQngr8Azg18BngXHgL/tYkyRpCKYcIbQT270rySlV9csB1CRJGoJu7ofw3CS3ALe2j5+V5Iy+VyZJGqhudhmdCvwJsBZ+e33CAf0sSpI0eF1NY11Vq5N0LvpNf8oZvNW/+NWwS5CkLUI3I4TVSZ4LVJJHJXkb7e6jXkryqiT/lOTLSV7e6+1Lkjatm0B4E/AWYBdgDFjaPp5SkjOT3JNk1YTlBya5LckdSd4BUFVfqqo3AscATmAvSQPWzS6j+6rq6M3c/lnAx4BPr1+QZFvgdOBlNAFzdZKvVNUt7Sp/3T4vSRqgbkYIq5JcnuR9SQ5O0vWNjKvqUuDnExbvC9xRVXdW1YPA54BXpvF+4GtVde1k20xyXJKVSVauWbOm21IkSVOYMhCqaglwJHAT8AqaGVCvn0GbuwCrOx6PtcveCrwUeE2SN22inhVVtayqlu20004zKEOS1GnKXUZJFgLPA54PPAu4GfjODNrMRpZVVZ0GnLaR5yRJA9DNMYQfA1cDp1TVpN/cp2EMeHLH44U0N+PpWpLlwPIlS5b0oBxJEmz6FprbVdXDwLOB/YGj2jOC/h34dlV9cjPbvBrYLcli4G7gCOCo6Wygqs4Fzl22bNkbN7MGDcjCdddMvVIfjM3bZyjtSqNsU8cQroLfXpn8KeCfgW8CLwDe3c3Gk5wNXAE8LclYkmPbkDkBuJDmeobPV9XNm/8WJEm90M0xhJXAo2mmvL4MOKCqftTNxqvqyEmWnw+cP406JUl9tqlA2DnJycA5wCPrlwGHJaGqPtz36ibhMQRp00696PahtHvSy3YfSrvqjU3tMtoW2AF4bPvfHYC5HT9DU1XnVtVx8+d3fUmEJGkKmxoh/LSq/s/AKpEkDdWmRggbu15AkjRLbSoQXjKwKqYpyfIkK8bHx4ddiiTNGpPuMqqqiXMQbTG8DkHTce26c4bS7t7znLRXo6Wbye0kSVsBA0GSBBgIkqTWSAaCB5UlqfdGMhC8ME2Sem8kA0GS1HsGgiQJMBAkSa2RDAQPKktS741kIHhQWZJ6r5t7Kksjx1t3StM3kiMESVLvGQiSJMBAkCS1DARJEjCigeBpp5LUeyMZCJ52Kkm9N5KBIEnqPQNBkgQYCJKkllcqS31y7bpzhtb23vMOH1rbGl2OECRJgIEgSWoZCJIkYEQDwQvTJKn3RjIQvDBNknpvJANBktR7BoIkCTAQJEktA0GSBHilstRT3stZo8wRgiQJMBAkSS0DQZIEGAiSpJaBIEkCRjQQnMtIknpvJAPBuYwkqfdGMhAkSb1nIEiSAANBktRy6gpJI+/Ui24fSrsnvWz3obTbL44QJEmAIwRpVrp23TlDavndQ2pXveAIQZIEGAiSpJaBIEkCPIYgqYfOuP6MIbX80iG1O7s4QpAkAQaCJKnlLiNJPXPFD9YOpd295w2l2VnHEYIkCXCEIM0KC9ddM+wSBmps3j7DLmFWcoQgSQK2oEBIsmuSTyb5wrBrkaStUV8DIcmZSe5JsmrC8gOT3JbkjiTvAKiqO6vq2H7WI0maXL9HCGcBB3YuSLItcDpwELAncGSSPftchyRpCn0NhKq6FPj5hMX7Ane0I4IHgc8Br+xnHZKkqQ3jGMIuwOqOx2PALkkWJPk48Owk75zsxUmOS7Iyyco1a9b0u1ZJ2moM47TTbGRZVdVa4E1TvbiqVgArAJYtW1Y9rk2StlrDGCGMAU/ueLwQ+MkQ6pAkdRjGCOFqYLcki4G7gSOAo6azgSTLgeVLlizpQ3mS1J1h3csZ+nM/536fdno2cAXwtCRjSY6tqoeBE4ALgVuBz1fVzdPZblWdW1XHzZ8/v/dFS9JWqq8jhKo6cpLl5wPn97NtSdL0bDFXKkuShmskAyHJ8iQrxsfHh12KJM0aIxkIHkOQpN4byUCQJPWegSBJAkb0BjlehyBpS3DtunOG2Pq7e77FkRwheAxBknpvJANBktR7BoIkCfAYgqRZYFD78heuu+Z3Hw+kVRibt89A2hnJEYLHECSp90YyECRJvWcgSJIAA0GS1DIQJEmAZxlJGkETz/ZRb4zkCMGzjCSp90YyECRJvWcgSJIAA0GS1DIQJEmAgSBJao1kICRZnmTF+Pj4sEuRpFljJAPB004lqfdGMhAkSb2Xqhp2DZstyRrgR5v58h2Be3tYziizLxr2wwb2RWO29sMfVNVOExeOdCDMRJKVVbVs2HVsCeyLhv2wgX3R2Nr6wV1GkiTAQJAktbbmQFgx7AK2IPZFw37YwL5obFX9sNUeQ5Ak/a6teYQgSepgIEiSgK0gEJIcmOS2JHckecdGnk+S09rnb0yy9zDq7Lcu+uHo9v3fmOS7SZ41jDoHYaq+6FjvOUl+k+Q1g6xvULrphyQvTHJ9kpuTfHvQNQ5KF/8+5ic5N8kNbV+8bhh19l1VzdofYFvgB8CuwKOAG4A9J6xzMPA1IMAfAd8bdt1D6ofnAr/X/n7QbOyHbvuiY71vAucDrxl23UP6m3g8cAvwlPbxzsOue4h98T+B97e/7wT8HHjUsGvv9c9sHyHsC9xRVXdW1YPA54BXTljnlcCnq3El8PgkTxp0oX02ZT9U1Xer6j/ah1cCCwdc46B08zcB8Fbg/wH3DLK4AeqmH44CvlhVPwaoqq25LwqYmyTADjSB8PBgy+y/2R4IuwCrOx6Ptcumu86om+57PJZm1DQbTdkXSXYBXg18fIB1DVo3fxO7A7+X5JIk1yR57cCqG6xu+uJjwB7AT4CbgBOr6pHBlDc42w27gD7LRpZNPM+2m3VGXdfvMcmLaAJh/75WNDzd9MVHgLdX1W+aL4SzUjf9sB2wD/AS4DHAFUmurKrb+13cgHXTF38CXA+8GHgqcFGSy6pqXZ9rG6jZHghjwJM7Hi+kSfjprjPqunqPSZ4JfAI4qKrWDqi2QeumL5YBn2vDYEfg4CQPV9WXBlLhYHT7b+Peqvol8MsklwLPAmZbIHTTF68D3lfNQYQ7kvwQeDpw1WBKHIzZvsvoamC3JIuTPAo4AvjKhHW+Ary2Pdvoj4DxqvrpoAvtsyn7IclTgC8C/30WfgPsNGVfVNXiqlpUVYuALwDHz7IwgO7+bXwZeH6S7ZI8FtgPuHXAdQ5CN33xY5qREkmeCDwNuHOgVQ7ArB4hVNXDSU4ALqQ5k+DMqro5yZva5z9OcxbJwcAdwP003wRmlS774X8BC4Az2m/GD9csnOWxy76Y9brph6q6NckFwI3AI8AnqmrV8Krujy7/Jv4GOCvJTTS7mN5eVbNuWmynrpAkAbN/l5EkqUsGgiQJMBAkSS0DQZIEGAiSpJaBoN/Rzu55fZJV7eyOj+947hlJvpnk9iT/nuTd7dwuJHlPkrdN2NZdSXZsf39iks8mubOdBuGKJK9un3thkvG23fU/L22fOzPJPUkmPd0xyclJbmlnar04yR9Mst59M+6gZjufSLLnRpYfk+RjG1n+6CTfaN/X4b2oQeoHA0ET/aqqllbVH9JM4PUWgCSPoblY531VtTvNFavPBY6faoNtaHwJuLSqdq2qfWgu/umcQO+ytt31P99ol58FHDhFE9cBy6rqmTQXkv19d29181TVG6rqlmm85NnA9u37OqdfdfVCkm2H1O6sviZqVBgI2pQr2DDJ11HA5VX1dYCquh84AZj0fgIdXgw82HnRV1X9qKo+OtULq+pSmmDa1DrfauuBGczU2o5yPpXk6+3o5tAkf5/kpiQXJNm+Xe+SJMva31/Xjpi+DTxvI9vcGfgXYGk7QnjqhOcvSXJqkkuT3JrmHgxfbEdgf9ux3p8nuardxj+u/+BO8g9JVqaZo/+9Heu/r2PU9MF22VnpuLfD+hFTO0L7VpLPAjcl2TbJB5Jc3b7+f2zkfT0uyVfT3B9g1fqRT1v/d9vlVyWZm2ROkn9u+/G6NPNlrR9R/WuSc4Gvt9s8s233uiQbm4VWfWQqa6PaD5yXAJ9sFz0DuKZznar6QZIdksybYnPPAK6dYp3nJ7m+4/FhVfWDaZS83kxnan0q8CJgT5pAPKyq/irJvwF/SjPSASDNNOnvpZkAbhz4Fs1o5beq6p4kbwDeVlWvmKTNB6vqgCQn0kwXsQ9NCP4gyanAzsDhwPOq6qEkZwBHA58G3lVVP2//f12cZj6qMZrZWp9eVZWO3X6bsC/wh1X1wyTH0Uzh8pwkjwYuT/L1qvphx/oHAj+pqj9t+2J+mmkfzgEOr6qr27+LXwEntn2xV5Kn03z4795u54+BZ7bv4RTgm1X1+rbmq5J8o51LSQNgIGiix7QfzItoAuCidnmYfBbYmuK535HkdJrZVB+sque0iy/bxAdmV5L8Oc3EdC+YwWa+1n7o3kQzjcEF7fKbaPqk037AJVW1pm3/HJopo6dr/bw5NwE3r59LK8mdNJOu7U8TElc3e994DBvu0/Bn7Qf4dsCTaILsFuAB4BNJvgqc10UNV3V84L8ceGbHaGI+sBvQGQg3AR9M8n7gvKq6LMlewE+r6mqA9TOBJtkf+Gi77PtJfsSGfrqoqtaPAF8OHJINx6LmAE9hds6ftEUyEDTRr6pqaZL5NB8kbwFOA24GDuhcMcmuwH1V9Z9J1tJ8IHWaC/yife1h6xdW1VvSHGxe2aui0xyEfhfwgqr69Qw29WuAqnokyUO1YW6XR9j4v5dezP2yvt5HOn7vbDPAp6rqnZ0vSrIYeBvwnKr6jyRnAXPauXn2pRnhHUGza+/FNDd02aZ9bWjuDrZe57fwAG+tqgsnK7iqbk+yD808YH+X5Os0o6eN9cem5hCf2O5hVXXbJtZXH3kMQRtVVePAXwBva/ed/19g/2w4++cxNEGx/gDupTTf7ua2zx8K3FBVv6G5FeWcJG/uaOKxM6kvyQlpJiQjybOBfwQOmXhXryTfn0k7U/ge8MIkC9o++m99audi4DXt8QiSPCHNmVTzaD5Qx9PMwHlQ+/wOwPyqOh/4S2Bpu527aEYa0NwRbPtJ2rsQeHM2HDPZPcnjOldI8vvA/VX1L8AHgb2B7wO/n+Q57Tpz0xwsvpRmFxftrqKnABv70L8QeGsbVuv/v2qAHCFoUlV1XZIbgCOq6jPtQb6Ptrt8tgU+Q3MnKarqxjSnXH4nSdHs0nhD+1wleRVwapK/AtbQfJC9vaO5iccQ/raqvpDkbOCFwI5JxoD/XVWfpJmL/vJ23Q/Q3NbwX9vPkh9X1SHtKKTz2+lj222s9+Gq+vAM+uenSd5Dc6zhpzTHSXp+lk5V3ZLkr2n2vW8DPAS8paquTHIdzQjsTjb0x1zgy0nm0Lz/k9rl/9Quv4omZCbbN/8Jmt1j17YfzmuAV01YZy/gA0keaet5c1U92B5c/mj7heFXwEuBM4CPt7vhHgaOqapf57/efOhvaG5OdGPb7l3AjHYjanqc7VQjKcl5wKHtPXAnW+cVwK5VddrgKpNGl4EgSQI8hiBJahkIkiTAQJAktQwESRJgIEiSWgaCJAmA/w//rDRn07/KagAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEGCAYAAACUzrmNAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAX+UlEQVR4nO3df5RVZb3H8c9HwMiESRFbLiYCQ0osJRi1TI1+UOgNf0D3JlpmUVxTy3R1K/updW/XfuqyZHlJzXTdlH6YiRHqNQ0zTQVBQa+GmjJqV6QcIiRBv/ePvUePs2Zm75k5Z++zZ96vtVjO2Wef/XyfAc/nPPvZ59mOCAEA0Jsdyi4AAND8CAsAQCbCAgCQibAAAGQiLAAAmYaXXcBA7LbbbjFhwoSyywCASlmxYsVTETG2L6+pdFhMmDBBd955Z9llAECl2H6kr6/hNBQAIBNhAQDIRFgAADJVes4CAMqwbds2tbe3a+vWrWWX0quRI0eqtbVVI0aMGPCxCAsA6KP29naNGjVKEyZMkO2yy+lWRGjjxo1qb2/XxIkTB3w8TkMBQB9t3bpVY8aMadqgkCTbGjNmTN1GP4QFAPRDMwdFp3rWSFgAADJVcs7C9mxJsydNmtT/g9z4n3WrJ9PbzyiuLQCFO+f6B+p6vNNmTs6130c+8hFdc8012n333bVmzZq61tBVJUcWEbEkIha0tLSUXQoAlOaEE07QsmXLCmmrkmEBAJAOPfRQ7brrroW0RVgAADIRFgCATJWc4K6cHJPpC5++e+DtTDykT7ufNPWkgbcJYEggLIawhasWltIuIQVUD2EBAAOU91LXeps3b55uuukmPfXUU2ptbdVZZ52l+fPnN6QtwgIAKuryyy8vrC0muAEAmQgLAEAmTkNhyKj3kgx5lXU+G6gnRhYAgEyEBQAgE2EBAMjEnAUKV9bcAdAw9b7lQY7bGqxfv17HH3+8/vznP2uHHXbQggULdOqpp9a3jhqEBQBU0PDhw/Wd73xH06ZN09/+9jdNnz5dM2fO1JQpUxrSXtOchrK9t+0LbP/M9sfLrgcAmtkee+yhadOmSZJGjRqlvffeW4899ljD2mtoWNi+2PaTttd02T7L9v2219n+nCRFxH0RcaKkf5HU1si6AGAw+dOf/qS77rpLBx54YMPaaPTI4hJJs2o32B4m6XxJh0maImme7Snpc0dI+p2kGxpcFwAMCps3b9bcuXN17rnnavTo0Q1rp6FhERHLJf2ly+YDJK2LiIci4llJV0g6Mt3/6og4SNJxjawLAAaDbdu2ae7cuTruuOM0Z86chrZVxgT3OEnrax63SzrQ9gxJcyS9TNLSnl5se4GkBZI0fvz4hhUJAM0sIjR//nztvffeOv300xveXhlh4W62RUTcJOmmrBdHxCJJiySpra0t6loZAPRHjktd6+2WW27RZZddpje+8Y2aOnWqJOnrX/+6Dj/88Ia0V0ZYtEt6dc3jVkmPl1AHAFTWwQcfrIjiPi+XcensHZL2sj3R9o6SjpF0dV8OYHu27UUdHR0NKRAA8FINHVnYvlzSDEm72W6X9JWIuMj2KZKulTRM0sURsbYvx42IJZKWtLW1fazeNQP1xmq3GAwaGhYRMa+H7UvVyyQ2AKC5sNwHCrdy0+JS2p02+v2ltAsMBk2z3EdfMGcBAMWqZFhExJKIWNDS0lJ2KQAwJHAaCgAGaOGqhXU93klTT8rcZ+edd9bmzZvr2m5vCIsh7NYHN5ZdAoCKqGRY2J4tafakSZPKLqW5PHxzn3Zv3fRMv5tqHz29368FUD3MWQAAMlUyLAAAxSIsAACZKhkWfM8CAIpVyQlu1oYC0EzyXOpab1u2bFFra+sLj08//fSG3teikmEBAEPd888/X2h7lTwNBQAoFmEBAMhEWABAPxR5l7r+qmeNlQwLroYCUKaRI0dq48aNTR0YEaGNGzdq5MiRdTleJSe4uRoKQJlaW1vV3t6uDRs2lF1Kr0aOHPmSK6YGopJhAQBlGjFihCZOnFh2GYWq5GkoAECxCAsAQCbCAgCQiTkLYJA65/oHSmn3tJmTS2kXjVXJkQWXzgJAsSoZFtz8CACKVcmwAAAUizkLDBkrNy0upd1po99fSrtAPTGyAABkIiwAAJkICwBAJsICAJCpkhPctmdLmj1p0qSySxmyWjetKKyt9tHTC2sLQPcqObLgexYAUKxKhgUAoFiEBQAgE2EBAMhEWAAAMlXyaqhGWvj03WWXAABNh5EFACATYQEAyERYAAAyERYAgEyEBQAgE2EBAMhUybCwPdv2oo6OjrJLAYAhoZJhwUKCAFCsSoYFAKBYhAUAIBNhAQDIxNpQAOrqnOsfKKXd02ZOLqXdoYKRBQAgE2EBAMjEaSg0vdZNKwppp3309ELaAaqIkQUAIBNhAQDIxGkooMFWblpcSrvTRr+/lHYxOBEWTWL908+UXQIA9IjTUACATJlhYfsVtndIf55s+wjbIxpfGgCgWeQZWSyXNNL2OEk3SPqwpEsaWRQAoLnkCQtHxBZJcyR9LyKOljSlsWUBAJpJrrCw/RZJx0n6VbqNiXEAGELyhMWpks6Q9IuIWGt7T0k31rsQ20fZ/oHtX9p+d72PDwDovzxh8aqIOCIiviFJEfGQpJvzHNz2xbaftL2my/ZZtu+3vc7259LjXhURH5N0giQuEAeAJpInLM7Iua07l0iaVbvB9jBJ50s6TMncxzzbtXMgX0yfBwA0iR7nHmwfJulwSeNsn1fz1GhJ2/McPCKW257QZfMBktalIxTZvkLSkbbvk3S2pF9HxMpe6logaYEkjR8/Pk8ZAIAB6m1k8bikOyVtlbSi5s/Vkt4zgDbHSVpf87g93fYJSe+S9D7bJ/b04ohYFBFtEdE2duzYAZQBAMirx5FFRKyWtNr2j9P9xkfE/XVo0903F+dJOq+b5wAAJcszZzFL0ipJyyTJ9lTbVw+gzXZJr6553KpkFJOb7dm2F3V0dAygDABAXnnC4kwl8wxPS1JErJI0YQBt3iFpL9sTbe8o6Rglp7Zyi4glEbGgpaVlAGUAAPLKExbbI6JfH+FtXy7pVkmvs91ue35EbJd0iqRrJd0n6ScRsbY/xwcAFCPPN7HX2D5W0jDbe0n6pKTf5zl4RMzrYftSSUtzVwkAKFWekcUnJO0j6R+SfiypQ9KnGlhTJuYsAKBYmWEREVsi4guSZkTE/hHxxYjYWkBtvdXEnAUAFCjP/SwOsn2vkvkF2d7P9sKGVwYAaBp5TkOdo+RLeBulF75/cWgjiwIANJdct1WNiPVdNj3XgFpyY84CAIqVJyzW2z5IUtje0fanlZ6SKgtzFgBQrDxhcaKkk5Ws39QuaWr6GAAwROT5nsXmiDiu4ZUAAJpW3i/l/Z+SGx4tl3RLf7/RDTSz1k0rCmurffT0wtoC6iEzLCJiku3xkg6R9F5JC20/HRFTG11cT2zPljR70qRJZZUAoMmcc/0DpbR72szJpbRbtMywsN0q6a1KwmI/SWsl/a7BdfUqIpZIWtLW1vaxMusAmtnKTYtLaXfaaO6KPBjlOQ31qJKVYr8eET3elAgAMHj1eDWU7c4geZOkSyUda/tW25fanl9IdQCAptDbyOJ2SdMiYrXtByU9qORU1AeUfIP7ogLqAwA0gTxzFndKepmSZclvlnRoRDzS6MIyamKCGwAK1NuX8na3fbqkxZIukfSApN0lzU23l4ZvcANAsXobWQyTtLMkF1QLAKBJ9RYWT0TEVwurBADQtHo7DcWIAgAgqfeRxTsLqwIYYlhaBFXT48giIv5SZCEAgOaV6+ZHzYabHwFAsSoZFlw6CwDFyrM2FADkxgKGg1MlRxYAgGIxsgAGuaKuvOKqq8GNkQUAIBNhAQDIRFgAADIRFgCATJUMC76UBwDFqmRY8KU8AChWJcMCAFAswgIAkImwAABkIiwAAJkICwBAJsICAJCJsAAAZCIsAACZCAsAQCbCAgCQqZJhwdpQAFCsSoYFa0MBQLEqGRYAgGIRFgCATIQFACATYQEAyERYAAAyDS+7AACoh5WbFpfU8pdKardYjCwAAJkICwBAJsICAJCJOQsAddG6aUVhbbWPnl5YW0gwsgAAZGJkAaByGMUUj5EFACATYQEAyERYAAAyERYAgExNM8Fte09JX5DUEhHvK6uO9U8/U1bTANC0GjqysH2x7Sdtr+myfZbt+22vs/05SYqIhyJifiPrAQD0T6NPQ10iaVbtBtvDJJ0v6TBJUyTNsz2lwXUAAAagoaehImK57QldNh8gaV1EPCRJtq+QdKSke/Mc0/YCSQskafz48fUrFgD64ZzrHyil3dNmTi60vTImuMdJWl/zuF3SONtjbF8g6U22z+jpxRGxKCLaIqJt7Nixja4VAKByJrjdzbaIiI2STiy6GABAtjJGFu2SXl3zuFXS4305gO3Zthd1dHTUtTAAQPfKCIs7JO1le6LtHSUdI+nqvhwgIpZExIKWlpaGFAgAeKlGXzp7uaRbJb3Odrvt+RGxXdIpkq6VdJ+kn0TE2kbWAQAYmEZfDTWvh+1LJS1tZNsAgPqp5HIfzFkAQLEqGRbMWQBAsSoZFgCAYhEWAIBMlQwL5iwAoFiVDAvmLACgWJUMCwBAsQgLAEAmwgIAkKmSYcEENwAUq5JhwQQ3ABSrkmEBACgWYQEAyERYAAAyVTIsmOAGgGJVMiyY4AaAYlUyLAAAxSIsAACZCAsAQCbCAgCQibAAAGSqZFhw6SwAFKuSYcGlswBQrEqGBQCgWIQFACATYQEAyERYAAAyERYAgEyEBQAgE2EBAMg0vOwC+sP2bEmzJ02aVHYpAIa4lZsWl9TylwptrZIjC76UBwDFqmRYAACKRVgAADIRFgCATIQFACATYQEAyERYAAAyERYAgEyEBQAgkyOi7Br6zfYGSY/08+W7SXqqjuVUzVDu/1DuuzS0+z+U+y692P/XRMTYvryw0mExELbvjIi2susoy1Du/1DuuzS0+z+U+y4NrP+chgIAZCIsAACZhnJYLCq7gJIN5f4P5b5LQ7v/Q7nv0gD6P2TnLAAA+Q3lkQUAICfCAgCQadCHhe1Ztu+3vc7257p53rbPS5+/2/a0MupshBx9Py7t8922f297vzLqbJSs/tfst7/t52y/r8j6Gi1P/23PsL3K9lrbvy26xkbJ8W+/xfYS26vTvn+4jDobwfbFtp+0vaaH5/v3nhcRg/aPpGGSHpS0p6QdJa2WNKXLPodL+rUkS3qzpD+UXXeBfT9I0i7pz4cNlr7n7X/Nfr+RtFTS+8quu+C//1dKulfS+PTx7mXXXWDfPy/pG+nPYyX9RdKOZddep/4fKmmapDU9PN+v97zBPrI4QNK6iHgoIp6VdIWkI7vsc6SkSyNxm6RX2t6j6EIbILPvEfH7iPhr+vA2Sa0F19hIef7uJekTkn4u6ckiiytAnv4fK+nKiHhUkiJisPwO8vQ9JI2ybUk7KwmL7cWW2RgRsVxJf3rSr/e8wR4W4yStr3ncnm7r6z5V1Nd+zVfyaWOwyOy/7XGSjpZ0QYF1FSXP3/9kSbvYvsn2CtvHF1ZdY+Xp+/cl7S3pcUn3SDo1Ip4vprzS9es9b3jDymkO7mZb12uF8+xTRbn7ZfvtSsLi4IZWVKw8/T9X0mcj4rnkA+agkqf/wyVNl/ROSS+XdKvt2yLigUYX12B5+v4eSaskvUPSayVdb/vmiNjU4NqaQb/e8wZ7WLRLenXN41YlnyT6uk8V5eqX7X0lXSjpsIjYWFBtRcjT/zZJV6RBsZukw21vj4irCqmwsfL+238qIv4u6e+2l0vaT1LVwyJP3z8s6exITuKvs/2wpNdLur2YEkvVr/e8wX4a6g5Je9meaHtHScdIurrLPldLOj69QuDNkjoi4omiC22AzL7bHi/pSkkfHASfJrvK7H9ETIyICRExQdLPJJ00SIJCyvdv/5eSDrE93PZOkg6UdF/BdTZCnr4/qmREJduvkvQ6SQ8VWmV5+vWeN6hHFhGx3fYpkq5VcoXExRGx1vaJ6fMXKLkK5nBJ6yRtUfKJo/Jy9v3LksZIWph+ut4eg2RFzpz9H7Ty9D8i7rO9TNLdkp6XdGFEdHu5ZZXk/Lv/mqRLbN+j5LTMZyNiUCxdbvtySTMk7Wa7XdJXJI2QBvaex3IfAIBMg/00FACgDggLAEAmwgIAkImwAABkIiwAAJkIC/QoXYl1le016Qqdr6x5bh/bv7H9gO0/2v5Sus6ObJ9p+9NdjvUn27ulP7/K9o9tP5QuM3Gr7aPT52bY7kjb7fzzrvS5XlfTTPc53fa96WqaN9h+TQ/7bR7wLyg5zoW2p3Sz/QTb369HGznr+Grn76mH508cRMt5oASEBXrzTERMjYg3KFmY7GRJsv1yJV/sOTsiJiv51u9Bkk7KOmAaKFdJWh4Re0bEdCVfmqpdxPDmtN3OP/+Tbr9E0qyMJu6S1BYR+yr5ot0383W1fyLioxFxbz2PaXtYP+r4cs3vqbvnL4iISwdWWf3ZHtTf9RpMCAvkdateXGzsWEm3RMR1khQRWySdIqnHe0bUeIekZ2u/FBcRj0TE97JemGM1TUXEjWk90gBW0k1HRz+yfV06Kppj+5u277G9zPaIdL+bbLelP384HWn9VtJbeznuZemo7I+2P5Zun2H7Rts/lnSP7WG2v2X7jnSU9K81x/hMWsdq22en2y5xej8O22fXjK6+XdPup9Ofp9q+LX3+F7Z3qenLN2zfnvbjkG7q38P28poR5yHp9lm2V6Y13ZBu29X2VWk7tzlZWqazlkW2r5N0qe2xtn+e9vUO293+7lAuUh2Z0k+675R0UbppH0kraveJiAdt72x7dMbh9pG0MmOfQ2yvqnk8NyIe7EPJnQa6ku5rJb1d0hQlYTk3Ij5j+xeS/knJCElS8iYq6SwlC/N1SLpRySinO/squY/AKyTdZftX6fYDJL0hIh62vUDJMgz7236ZpFvSN9fXSzpK0oERscX2rrUHTh8fLen1ERGuOXVY41JJn4iI39r+qpJv+H4qfW54RBxg+/B0e9dTW8dKujYi/iP9d7GT7bGSfiDp0LT2zprOknRXRBxl+x1pu1PT56ZLOjginkkD8pyI+J2TJWiuVbIiLJoIYYHevDx9056gJByuT7dbPa9SGRnPvYTt85WsdvtsROyfbr45It7bz5o7j/sBJQsFvm0Ah/l1RGxzsiTEMEnL0u33KPmd1DpQ0k0RsSFtf7GSJcC788uIeEbSM7ZvVBIST0u6PSIeTvd5t6R9/eLd+1ok7aXkzfuHnaOniOg60tokaaukC9MQuqb2Sdstkl4ZEZ13xfuRpJ/W7HJl+t8V3fRRStZdujgdWV0VEatsz1ByWvHhLjUdLGluuu03tsek7UvS1envQGmfpvjFlX9H2x4VEX/rpn2UhNNQ6M0zETFV0muU3HHs5HT7WiVvxC+wvaekzen/4Bsl7dLlWKOUvCGuVXIXL0lSRJysZNQytl5FpxO9X5B0RET8YwCH+ockpfc52BYvro3zvLr/oJV37Zyu+3U+/nvNNiv59N85bzMxPe3XW1ArIrYrCZ+fKxmBLOtp3x50/r6eUzd9TE8FHirpMUmXpZPmPdXU21LYtX3dQdJbavo6jqBoPoQFMkVEh6RPSvp0+onyvyUd7BevUnq5pPP04mTycklH2B6VPj9H0uqIeE7JLUxH2v54TRM7DaQ+26c4WThOtt8k6b+UBMWTXfb734G0k+EPkmakn55HSPrnXvY90vZI22OULPh2Rzf7XCvp4zVzI5Ntv0LSdZI+4mSV2M7TTi+wvbOklohYquTU0tTa59O/y7/WzEd8UFLue287ubrsyYj4gZLTktOUnKJ7m+2JXWpaLum4dNsMJcuhd3e/iOuUzHl1tjG1m31QMk5DIZeIuMv2aknHRMRlto+U9L30NNIwSZcpufuYIuJuJ5eN/s52KLll6UfT58L2UZLOsf0ZSRuUfMr8bE1zXecs/j0ifuZuVtOMiIuUnMe/Jd33W0puk/nT9LTGoxFxhJPLdms/6e6UHqPTdyPiuwP4/Txh+0wlb5xPKJmX6emqptsl/UrSeElfi4jHbXc9ZXWhktNAK510ZIOkoyJiWfpmeqftZ5WsIPr5mteNkvRL2yOV9Pe0btr/kKQL0sB5SH1baXmGpH+zvU3SZknHR8SGdI7lSts7KPn7ninpTEk/tH23ktVNP9TDMT8p6fx0v+FKQubEPtSEArDqLCrP9jWS5kRyv+We9nmvpD0j4rziKuu2jjOVnK77dpl1AH3FyAKVl2cyPCKuydoHQM8YWQAAMjHBDQDIRFgAADIRFgCATIQFACATYQEAyPT/HtZtAF8HlC8AAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.hist(rouge1_mid_fmeasure_base, alpha=0.5)\n",
    "plt.hist(rouge2_mid_fmeasure_base, alpha=0.5)\n",
    "plt.hist(rougeL_mid_fmeasure_base, alpha=0.5)\n",
    "plt.xlabel('ROUGE1,2,L mid f measure score')\n",
    "plt.ylabel('Tweets')\n",
    "plt.yscale(\"log\")\n",
    "plt.legend(['1','2','L'])\n",
    "plt.show()\n",
    "\n",
    "plt.hist(rouge1_mid_precision_base, alpha=0.5)\n",
    "plt.hist(rouge2_mid_precision_base, alpha=0.5)\n",
    "plt.hist(rougeL_mid_precision_base, alpha=0.5)\n",
    "plt.xlabel('ROUGE1,2,L mid precision score')\n",
    "plt.ylabel('Tweets')\n",
    "plt.yscale(\"log\")\n",
    "plt.legend(['1','2','L'])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "edb66246-71d6-412e-89f1-2db8ad29dc0f",
   "metadata": {},
   "source": [
    "# Subjects"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9283ba36",
   "metadata": {},
   "source": [
    "In this section we investigate the subject distribution . After collecting the data, I wanted to check the distribution of arXiv sections (e.g. hep, astro, cs, etc.). Since I didn't save the entire arXiv pages and I didn't want to run the Twitter scan from scrach, I wrote below a code that obtains this information, resaves the full_data.csv as full_data_subjects.csv, and plots a few statistics."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "636aaf6d-370c-4483-80b5-75f47d9985d0",
   "metadata": {},
   "outputs": [],
   "source": [
    "# parsing functions, slightly modified from data_pull/arxiv_utils.py\n",
    "from html.parser import HTMLParser\n",
    "import requests\n",
    "\n",
    "class arXivHTMLParser(HTMLParser):\n",
    "    '''\n",
    "    Based on https://docs.python.org/3/library/html.parser.html\n",
    "    '''\n",
    "    def __init__(self):\n",
    "        HTMLParser.__init__(self)\n",
    "        \n",
    "        self.title = ''\n",
    "        self.authors = []\n",
    "        self.abstract = ''\n",
    "        \n",
    "        self.title_flag = False\n",
    "        self.authors_flag = False\n",
    "        self.abstract_flag = False\n",
    "        self.metadata = []\n",
    "        self.metatable_flag = False\n",
    "        \n",
    "    def handle_starttag(self, tag, attrs):\n",
    "        # print(\"Encountered a start tag:\", tag)\n",
    "        if tag == 'title' and self.title == '' :\n",
    "            self.title_flag = True\n",
    "        for attr in attrs:\n",
    "            if attr == ('class', 'metatable'):\n",
    "                self.metatable_flag = True\n",
    "\n",
    "    def handle_endtag(self, tag):\n",
    "        # print(\"Encountered an end tag :\", tag)\n",
    "        if tag == 'title':\n",
    "            self.title_flag = False\n",
    "            \n",
    "        if self.authors_flag and tag == 'div':\n",
    "            self.authors_flag = False\n",
    "        \n",
    "        if self.abstract_flag and tag == 'blockquote':\n",
    "            self.abstract_flag = False\n",
    "        \n",
    "    def handle_data(self, data):\n",
    "        # print(\"Encountered some data  :\", data)\n",
    "        if self.title_flag:\n",
    "            self.title = data\n",
    "            \n",
    "        if data == 'Authors:':\n",
    "            self.authors_flag = True\n",
    "        \n",
    "        if self.authors_flag and data != 'Authors:' and data != ', ':\n",
    "            self.authors.append(data)\n",
    "\n",
    "        if data == 'Abstract:':\n",
    "            self.abstract_flag = True\n",
    "            \n",
    "        if data != 'Abstract:' and self.abstract_flag:\n",
    "            self.abstract = self.abstract + data\n",
    "\n",
    "        if self.metatable_flag:\n",
    "            if data == 'Which authors of this paper are endorsers?':\n",
    "                self.metatable_flag = False\n",
    "            if data != '\\n':\n",
    "                self.metadata.append(data)\n",
    "\n",
    "            \n",
    "def get_arXiv_details(url):\n",
    "    '''\n",
    "        This method uses the above HTML parser (arXivHTMLParser)\n",
    "        accepts a url of the variety 'https://arxiv.org/abs/xxxx.xxxxx'\n",
    "        returns the [title, authors, abstracts]\n",
    "        \n",
    "        does not support comments, journal, and other information -- but can be extended to support it by modifying this method and the parser\n",
    "    '''\n",
    "    url = url.replace('arxiv','export.arxiv') # arxiv asks bots to use this website, see https://arxiv.org/denied.html ; otherwise it blocks the bot often\n",
    "    \n",
    "    req = requests.get(url, 'html.parser')\n",
    "    parser = arXivHTMLParser()\n",
    "    parser.feed(req.text)\n",
    "    \n",
    "    parser.title = parser.title[13:] # in title: remove [xxxx.xxxxx] prefix\n",
    "    parser.abstract = parser.abstract.replace('\\n',' ')[1:] # replace \\n by space and remove two spaces in the beginning ('  '). Used to be 2: with arxiv.org rather than export.arxiv.org\n",
    "    \n",
    "    # remove extra entries from author list. Was unnecessary in arxiv.org, became necessary with export.arxiv.org\n",
    "    try:\n",
    "        parser.authors.remove('\\n')\n",
    "    except:\n",
    "        pass\n",
    "    try:\n",
    "        while True:\n",
    "            parser.authors.remove(', \\n')\n",
    "    except ValueError:\n",
    "        pass\n",
    "    \n",
    "    return parser.title, parser.authors, parser.abstract, parser.metadata"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "adf855f2-30ba-472f-9ab4-df8acfdf9538",
   "metadata": {},
   "outputs": [],
   "source": [
    "# dedicated functions for parsing subjects\n",
    "import re\n",
    "\n",
    "def get_subjects_from_line(line):\n",
    "    listparse = re.findall(r'\\([^()]*\\)', line)\n",
    "    return [listparse[i][1:-1] for i in range(len(listparse))]\n",
    "    \n",
    "def get_subjects(metadata):\n",
    "    for i, line in enumerate(metadata):\n",
    "        if line == 'Subjects:':\n",
    "            break\n",
    "    subject_main = metadata[i+1]\n",
    "    subject_main = get_subjects_from_line(subject_main)\n",
    "\n",
    "    subject_cross = ''\n",
    "    if metadata[i+2][0] == ';':\n",
    "        subject_cross = metadata[i+2]\n",
    "    subject_cross = get_subjects_from_line(subject_cross)\n",
    "\n",
    "    for subject in subject_cross:\n",
    "        subject_main.append(subject)\n",
    "\n",
    "    return subject_main"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "bde1b645",
   "metadata": {},
   "outputs": [],
   "source": [
    "# load data\n",
    "import pandas as pd\n",
    "df = pd.read_csv('data/full_data.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "b8dfa518-e5a8-4d7b-9a7b-0d14a4396a28",
   "metadata": {},
   "outputs": [],
   "source": [
    "# actual re-run on arxiv, very long run-time, better skip and load from file\n",
    "metadata_list = []\n",
    "df_arxiv_subjects = []\n",
    "i = 0\n",
    "while True:\n",
    "    try:\n",
    "        title, authors, abstract, metadata = get_arXiv_details(df['arxiv_link'].iloc[i])\n",
    "        metadata = [metadata[i].replace('\\n','') for i in range(len(metadata))]\n",
    "        metadata_list.append(metadata)\n",
    "        df_arxiv_subjects.append(get_subjects(metadata))\n",
    "        i += 1\n",
    "    except:\n",
    "        continue\n",
    "    if i == len(df):\n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "8ab55be0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>TweetID</th>\n",
       "      <th>AuthorID</th>\n",
       "      <th>AuthorName</th>\n",
       "      <th>Tweets</th>\n",
       "      <th>arxiv_link</th>\n",
       "      <th>Abstract</th>\n",
       "      <th>Title</th>\n",
       "      <th>Thread_length</th>\n",
       "      <th>Tweets_coarse</th>\n",
       "      <th>arxiv_subjects</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>695565872891621376</td>\n",
       "      <td>75249390</td>\n",
       "      <td>Axel Maas</td>\n",
       "      <td>I have published a new blog entry on our recen...</td>\n",
       "      <td>http://arxiv.org/abs/1601.02006</td>\n",
       "      <td>Observable states are gauge-invariant. In a no...</td>\n",
       "      <td>Gauge invariance and the physical spectrum in ...</td>\n",
       "      <td>1</td>\n",
       "      <td>['I have published a new blog entry on our rec...</td>\n",
       "      <td>[hep-ph]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>694223402932899841</td>\n",
       "      <td>212431150</td>\n",
       "      <td>Faryad Sahneh</td>\n",
       "      <td>Our new paper: \"Sequential Monte Carlo Filteri...</td>\n",
       "      <td>http://arxiv.org/abs/1601.07606</td>\n",
       "      <td>We use a multivariate formulation of sequentia...</td>\n",
       "      <td>Sequential Monte Carlo Filtering Estimation of...</td>\n",
       "      <td>1</td>\n",
       "      <td>['Our new paper: \"Sequential Monte Carlo Filte...</td>\n",
       "      <td>[stat.AP, q-bio.PE]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>694093392414937088</td>\n",
       "      <td>2999702157</td>\n",
       "      <td>Anton Ilderton</td>\n",
       "      <td>New paper on measuring vacuum birefringence us...</td>\n",
       "      <td>http://arxiv.org/abs/1601.08045</td>\n",
       "      <td>The measurement of vacuum polarisation effects...</td>\n",
       "      <td>Prospects for studying vacuum polarisation usi...</td>\n",
       "      <td>1</td>\n",
       "      <td>['New paper on measuring vacuum birefringence ...</td>\n",
       "      <td>[hep-ph, physics.optics]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>694076172372697088</td>\n",
       "      <td>1430303833</td>\n",
       "      <td>Dave Woods</td>\n",
       "      <td>New paper on arXiv on screening for GLMs with ...</td>\n",
       "      <td>http://arxiv.org/abs/1601.08088</td>\n",
       "      <td>The first investigation is made of designs for...</td>\n",
       "      <td>Model selection via Bayesian information capac...</td>\n",
       "      <td>1</td>\n",
       "      <td>['New paper on arXiv on screening for GLMs wit...</td>\n",
       "      <td>[stat.ME]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0             TweetID    AuthorID      AuthorName  \\\n",
       "0           0  695565872891621376    75249390       Axel Maas   \n",
       "1           1  694223402932899841   212431150   Faryad Sahneh   \n",
       "2           2  694093392414937088  2999702157  Anton Ilderton   \n",
       "3           3  694076172372697088  1430303833      Dave Woods   \n",
       "\n",
       "                                              Tweets  \\\n",
       "0  I have published a new blog entry on our recen...   \n",
       "1  Our new paper: \"Sequential Monte Carlo Filteri...   \n",
       "2  New paper on measuring vacuum birefringence us...   \n",
       "3  New paper on arXiv on screening for GLMs with ...   \n",
       "\n",
       "                        arxiv_link  \\\n",
       "0  http://arxiv.org/abs/1601.02006   \n",
       "1  http://arxiv.org/abs/1601.07606   \n",
       "2  http://arxiv.org/abs/1601.08045   \n",
       "3  http://arxiv.org/abs/1601.08088   \n",
       "\n",
       "                                            Abstract  \\\n",
       "0  Observable states are gauge-invariant. In a no...   \n",
       "1  We use a multivariate formulation of sequentia...   \n",
       "2  The measurement of vacuum polarisation effects...   \n",
       "3  The first investigation is made of designs for...   \n",
       "\n",
       "                                               Title  Thread_length  \\\n",
       "0  Gauge invariance and the physical spectrum in ...              1   \n",
       "1  Sequential Monte Carlo Filtering Estimation of...              1   \n",
       "2  Prospects for studying vacuum polarisation usi...              1   \n",
       "3  Model selection via Bayesian information capac...              1   \n",
       "\n",
       "                                       Tweets_coarse            arxiv_subjects  \n",
       "0  ['I have published a new blog entry on our rec...                  [hep-ph]  \n",
       "1  ['Our new paper: \"Sequential Monte Carlo Filte...       [stat.AP, q-bio.PE]  \n",
       "2  ['New paper on measuring vacuum birefringence ...  [hep-ph, physics.optics]  \n",
       "3  ['New paper on arXiv on screening for GLMs wit...                 [stat.ME]  "
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# insert new list into dataframe\n",
    "df.insert(len(df.columns), 'arxiv_subjects', df_arxiv_subjects)\n",
    "df.head(4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "96a034dc",
   "metadata": {},
   "outputs": [],
   "source": [
    "# uncomment to save again. Generating this dataset takes a long time, so commented to avoid overwrite in a mass run\n",
    "# df.to_csv('data/full_data_subjects.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "924df535",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>Unnamed: 0.1</th>\n",
       "      <th>TweetID</th>\n",
       "      <th>AuthorID</th>\n",
       "      <th>AuthorName</th>\n",
       "      <th>Tweets</th>\n",
       "      <th>arxiv_link</th>\n",
       "      <th>Abstract</th>\n",
       "      <th>Title</th>\n",
       "      <th>Thread_length</th>\n",
       "      <th>Tweets_coarse</th>\n",
       "      <th>arxiv_subjects</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>695565872891621376</td>\n",
       "      <td>75249390</td>\n",
       "      <td>Axel Maas</td>\n",
       "      <td>I have published a new blog entry on our recen...</td>\n",
       "      <td>http://arxiv.org/abs/1601.02006</td>\n",
       "      <td>Observable states are gauge-invariant. In a no...</td>\n",
       "      <td>Gauge invariance and the physical spectrum in ...</td>\n",
       "      <td>1</td>\n",
       "      <td>['I have published a new blog entry on our rec...</td>\n",
       "      <td>[hep-ph]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>694223402932899841</td>\n",
       "      <td>212431150</td>\n",
       "      <td>Faryad Sahneh</td>\n",
       "      <td>Our new paper: \"Sequential Monte Carlo Filteri...</td>\n",
       "      <td>http://arxiv.org/abs/1601.07606</td>\n",
       "      <td>We use a multivariate formulation of sequentia...</td>\n",
       "      <td>Sequential Monte Carlo Filtering Estimation of...</td>\n",
       "      <td>1</td>\n",
       "      <td>['Our new paper: \"Sequential Monte Carlo Filte...</td>\n",
       "      <td>[stat.AP, q-bio.PE]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>694093392414937088</td>\n",
       "      <td>2999702157</td>\n",
       "      <td>Anton Ilderton</td>\n",
       "      <td>New paper on measuring vacuum birefringence us...</td>\n",
       "      <td>http://arxiv.org/abs/1601.08045</td>\n",
       "      <td>The measurement of vacuum polarisation effects...</td>\n",
       "      <td>Prospects for studying vacuum polarisation usi...</td>\n",
       "      <td>1</td>\n",
       "      <td>['New paper on measuring vacuum birefringence ...</td>\n",
       "      <td>[hep-ph, physics.optics]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>694076172372697088</td>\n",
       "      <td>1430303833</td>\n",
       "      <td>Dave Woods</td>\n",
       "      <td>New paper on arXiv on screening for GLMs with ...</td>\n",
       "      <td>http://arxiv.org/abs/1601.08088</td>\n",
       "      <td>The first investigation is made of designs for...</td>\n",
       "      <td>Model selection via Bayesian information capac...</td>\n",
       "      <td>1</td>\n",
       "      <td>['New paper on arXiv on screening for GLMs wit...</td>\n",
       "      <td>[stat.ME]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>693490180162977792</td>\n",
       "      <td>180240998</td>\n",
       "      <td>Franck Marchis</td>\n",
       "      <td>New @PlanetImager paper \"The PDS 66 Circumstel...</td>\n",
       "      <td>https://arxiv.org/abs/1601.07248</td>\n",
       "      <td>We present H and K band imaging polarimetry fo...</td>\n",
       "      <td>The PDS 66 Circumstellar Disk as seen in Polar...</td>\n",
       "      <td>1</td>\n",
       "      <td>['New @PlanetImager paper \"The PDS 66 Circumst...</td>\n",
       "      <td>[astro-ph.EP]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0  Unnamed: 0.1             TweetID    AuthorID      AuthorName  \\\n",
       "0           0             0  695565872891621376    75249390       Axel Maas   \n",
       "1           1             1  694223402932899841   212431150   Faryad Sahneh   \n",
       "2           2             2  694093392414937088  2999702157  Anton Ilderton   \n",
       "3           3             3  694076172372697088  1430303833      Dave Woods   \n",
       "4           4             4  693490180162977792   180240998  Franck Marchis   \n",
       "\n",
       "                                              Tweets  \\\n",
       "0  I have published a new blog entry on our recen...   \n",
       "1  Our new paper: \"Sequential Monte Carlo Filteri...   \n",
       "2  New paper on measuring vacuum birefringence us...   \n",
       "3  New paper on arXiv on screening for GLMs with ...   \n",
       "4  New @PlanetImager paper \"The PDS 66 Circumstel...   \n",
       "\n",
       "                         arxiv_link  \\\n",
       "0   http://arxiv.org/abs/1601.02006   \n",
       "1   http://arxiv.org/abs/1601.07606   \n",
       "2   http://arxiv.org/abs/1601.08045   \n",
       "3   http://arxiv.org/abs/1601.08088   \n",
       "4  https://arxiv.org/abs/1601.07248   \n",
       "\n",
       "                                            Abstract  \\\n",
       "0  Observable states are gauge-invariant. In a no...   \n",
       "1  We use a multivariate formulation of sequentia...   \n",
       "2  The measurement of vacuum polarisation effects...   \n",
       "3  The first investigation is made of designs for...   \n",
       "4  We present H and K band imaging polarimetry fo...   \n",
       "\n",
       "                                               Title  Thread_length  \\\n",
       "0  Gauge invariance and the physical spectrum in ...              1   \n",
       "1  Sequential Monte Carlo Filtering Estimation of...              1   \n",
       "2  Prospects for studying vacuum polarisation usi...              1   \n",
       "3  Model selection via Bayesian information capac...              1   \n",
       "4  The PDS 66 Circumstellar Disk as seen in Polar...              1   \n",
       "\n",
       "                                       Tweets_coarse            arxiv_subjects  \n",
       "0  ['I have published a new blog entry on our rec...                  [hep-ph]  \n",
       "1  ['Our new paper: \"Sequential Monte Carlo Filte...       [stat.AP, q-bio.PE]  \n",
       "2  ['New paper on measuring vacuum birefringence ...  [hep-ph, physics.optics]  \n",
       "3  ['New paper on arXiv on screening for GLMs wit...                 [stat.ME]  \n",
       "4  ['New @PlanetImager paper \"The PDS 66 Circumst...             [astro-ph.EP]  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# reload modified data\n",
    "import ast\n",
    "import pandas as pd\n",
    "df = pd.read_csv('data/full_data_subjects.csv')\n",
    "df['arxiv_subjects'] = df['arxiv_subjects'].apply(lambda x: ast.literal_eval(x))\n",
    "df.head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ef0b5c80",
   "metadata": {},
   "source": [
    "Statistics of subjects"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "19fce44d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAa4AAAD4CAYAAAC0VQLEAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAseUlEQVR4nO3debhlVWHm/+9LKcg8CI1QoVNAE2kGKaEAERRQbHAIoDJKGghpaAyGQKfaQIho+ic/RW2DEiNBIkWQAAoUIFNQpChBqrCAGlFUBpXBoRhKkEEo3v5jrUMdbp1zp7rn3nvOfT/Pc597zh7W3uvCU+tZe6/1LtkmIiKiW6w21jcQERExFGm4IiKiq6ThioiIrpKGKyIiukoaroiI6CqvG+sb6HUbb7yxp0yZMta3ERHRVe6+++6ltjdptS8NV4dNmTKFefPmjfVtRER0FUk/b7cvjwr7kPRsm+1/JmmhpCWSFki6QNIGo3x7ERETXnpcgyDpAOBU4L22H5U0CTgG2BR4ur9zFz26jCmnXd/5m4yIGEce/uz7O1Z2Tzdcko4GpgMGFgLXAp8ElgPLbL9zkEWdAUy3/SiA7eXA10f+jiMiYiA923BJ2p7S4Oxpe6mkjYDbgP1rr2mDIRS3PXDPEK59AnACwKT1Wr5bjIiIYerld1zvAq6wvRTA9pPAHcAMSccDk4ZTqKQdJc2X9ICkw1sdY/t829NsT5u01vrDvf+IiGihlxsuUR4Rvsr2icDfA1sA8yW9cZBlLQF2rmUssj0VuBFYc8TuNiIiBqVnHxUCtwAzJf2j7Sfqo8INbc8F5kr6U0oD9sQgyvoM8AVJB9l+pG4bVKO14+T1mdfBl5QRERNNzzZctpdIOgu4TdJy4F5gPUnbUHpjtwALJG0OXGD7ffXUtSQ90lTUF21/UdImwI11ROHTwGLgP0arPhERUSjrcXXWtGnTnAnIERFDI+lu29Na7evld1wREdGD0nC1IOlNki6rIwfvk3SDpD+RNEXS4rG+v4iIiaxn33ENlyQBM4GLbB9Rt02lpGT8cqjlJTkjYmR1MpEhukPPN1zDSM/YF3jJ9nmNDbbn17KmjMItR0REP3q64RpmesYOwN2reN0kZ0REdEivv+PqSHrGQJKcERHROb3ecA0nPWMJsMvo3F5ERAxVT8/jqo8KZwJ79EnPeKDuvxf488Y7rLpNwBzKpOSv1W27AmsBPweus73DYO8h87giIoauv3lcPf2OazjpGbYt6YPAOZJOA14AHgZOqcW+uU+yxqm2vzVKVYqImPB6uuECsH0RcNEAhz0GNCKfsP0YcFibY18/QrcWERHD0OvvuCIioseMi4ZL0rH1cV2nyn92kMftJmmWpJ9KukfS9ZJ27HPMAkmXduZOIyJiIOPlUeGxlLT1x/rukDTJ9vJO34CkTYFvAh+x/YO6bS9ga2BR/f5fKY39OyWtbfv3A5Wb5IyIoUs6RvSnoz0uSVdLulvSEkknSJokaYakxZIWSTpV0iHANOCSurLwmpIelnSmpNuBQyUdWY9fLOnsNtfaR9JsSTNrvuB5klZr2n9W7S3NqY1UXx+jxDz9oLHB9u22r2465iPAxcDNwIGr/heKiIih6vSjwuNs70JpmE4GpgKTbe9ge0fgQttXAPOAo2xPtf18PfcF23sBs4GzKZOJpwK7Sjq4zfV2A/4G2JHSU/pQ3b42MMf2TrW841ucuz1wzwD1ORy4HLgUOLLdQbWRnidp3vLnlg1QZEREDEWnG66TJS2gzIvaAlgd2ErSuZIOAH7Xz7mX19+7ArNs/9b2y8AlQN98wYa7bD9YHy1eCuxVt/8BuK5+vhuYMtCNS5or6UeSvlS/7wr81vbPKcPod5a0Yatzk5wREdE5HWu4JO0D7EeZ/LsTZQ7VGsBOwCzgJOCCfopovD9Sm/J3r48W50tqPLbrO5u68f0lr5hpvZzW7/aWADu/eqK9O/AJoNHyHAlsK+lh4AFgPeDD/dx/RER0QCcHZ6wPPGX7OUnbAm8DNgZWs32lpAeAGfXYZ4B125QzF/iSpI2BpygNyLm251IeHQKvNpS7SdqSknBxOHD+EO73K8BcSf/R9J5rrVr2asChwFtsP1q37UuJjuqv8WXHyeszLy+aIyJGTCcbrpuAEyUtBO6nPC6cDMxqGjRxev09AzhP0vPAHs2F2H5c0unArZTe1w22r2lzzTuBz1Lecc2mxD21VXtq02yfaftXkg4HzpY0GfgNsBT4P5RHk482Gq1qNrCdpM1sPz7A3yIiIkZIz2QV1h7XdNsfGONbeY1kFUZEDF1/WYXjYgJyRETEYPVMj2ukSHoTcA5lNOOLrAjY/Q/gANv3Nx17DvCY7c+1K2+NzbbxZsec07H7jYkhE3JjokmPa5DqkiYzKcPvt7a9HfB3wKbAZcARTceuBhzCimH7ERExCsZL5FNHSDoamE4ZFr8QuBb4JGVI/DLbfeeD7UsZOn9eY0NjrS5JyyiN1D/UXe8EHq7zuiIiYpT0bMNVF5E8A9jT9tK6iORtwP62H5W0QYvTdqBMUF6J7YWSXpG0k+0FlN5Xy7BdSScAJwBMWm+TVa9MRES8qpcfFb4LuML2UgDbTwJ3ADMkHQ9MGkaZlwJHSHodcBDQcgHJJGdERHROz/a4KHO+XjPyxPaJknYH3g/MlzTV9hNNhyyhvLdq51JKwO5twELbvxnoJjIBOSJiZPVyj+sW4DBJbwSQtJGkrW3PtX0mZXLxFn3O+R6wRu2RUc/bVdLeALYfAJ6gTHLOmlwREWOgZxsu20uAs4DbatDvF4HPN5ZHoSRfLJC0uaQb6jkGPgi8R9IDkpYAn+K164RdCmzLAKkcERHRGZnH1WFJzoiIGLrM44qIiJ4xrgZnSDoWuNn2YwMdO8zyn7W9zgDH7ANcAzzUtHm67e9KWg4sovzdfgQcY/u5/spb9Ogyppx2/Srdd0wcSciIGNh463EdC2zeaoek4QxfH67v19WYGz/frdufr993oCxOeeIo3lNERDBKDZekqyXdLWlJXdZ+kqQZkhbXwRKnSjoEmAZcUheHXFPSw5LOlHQ7cKikIxuDKySd3eZa+0iaLWmmpPsknde0jAqSzpK0QNIcSZuuQrW+D/yXVTg/IiKGYbR6XMfZ3oXSMJ1MWQBysu0dbO8IXGj7CmAecFTt1Txfz33B9l6UUYBnUyYWTwV2lXRwm+vtBvwNZV2urYEP1e1rA3PqisyzgeNbn847mlZXni9p6+addQLyeymPDVdSG+d5kuYtf25Z+79KREQM2Wg1XCfXIelzKHOnVge2knSupAOA3/VzbiPEdldK+O1vbb8MXELJC2zlLtsP2l5OGb6+V93+B+C6+vluYEqb8/s+Knygbl9T0nxKA/sL4F9bnZzkjIiIzun44Iw62GE/YA/bz0maBawB7ATsD5wEHAYc16aI3zeKalP+7sC/1K9nUhrBvmP8G99f8orx/8sZev2ftz11KCckOSMiYmSNRo9rfeCp2mhtC7wN2BhYzfaVwCeAneuxzwDrtilnLrC3pI3rQI0jgdtqEkajZ3RtPXY3SVvWd1uHA7d3qG4RETHKRmM4/E3AiZIWAvdTHhdOBmY1DZo4vf6eAZwn6Xlgj+ZCbD8u6XTgVkrv6wbb17S55p2UWKYdKe+y+k25kHQgMK1GQUF9x9V0yKfrO7iIiBhjPZecUR9NTrf9gTG+FSDJGRERw5HkjIiI6Bk91+MaDEmnAOcPlHrR55wpwHV18vGgrbHZNt7smHOGdH/R3ZJ+EbHq0uNa2SnAWmN9ExERMXTjsuGSdIak+yV9V9KlkqZLmiVpWt2/saSH6+cpkr4v6Z768/a6fZ96zhWSfizpEhUnU2KlbpV0a4trHyvpGkk31Xv4ZNPuSZK+VhNAbpa0Zuf/GhER0WzcNVySdgGOAN5KSbzYdYBTfgO8x/bOlKHvX27a91ZK72o7YCtgT9tfpqyvta/tfduUuRtwFCWh49BGgwlsA3zF9vbA08CH29QhyRkRER0y7hou4B3ATNvP2f4dcO0Ax78e+JqkRcC3KI1Uw122H7H9CjCf9kkZfX3H9hM1duoqViRvPGR7fv3cNnkjyRkREZ0zrpY1adJqxMjLrGho39C0/VTg15QkjtWAF5r2vdj0uWVShqQPAo3Hgf+jzfUb3/uWN+CjwiRnRESMrPHY45oNfLCmw68L/Gnd/jCwS/18SNPx6wOP117VfwcGs/zJqwkdtmc2JW80Jly9R9JG9R3WwcAdq1KhiIgYOeOu4bJ9DyVYdz5wJWX5EIAvAB+V9ANKZFTDPwPHSJoD/Akrsg37cz5wY6vBGdXtwMWNe2hq0CIiYoyN+3lckj4FPGv7C6N0vWMp8U8fG4nykpwRETF0XTGPqw5Db7n68QiV/+wgjtmHMgqxeduMusgldXj9/U3rdCW/MCJilI2nwRnHAospQ9VfZftTNQ1+tDwyQG/rqKE8Olz06DKmnHb9CNxWjKSkW0R0r473uCRdLenuOmn3BEmTai9msaRFkk6tPZppwCW1J7OmpIclnSnpdspcqiPr8Yslnd3mWvtImi1ppqT7JJ3XlECPpLMkLZA0R9Kmna57RESMvNHocR1n+8k6Qu+HlPlPkxuZf5I2sP20pI9RUt3n1e0AL9jeqz5CnEMZVfgUcLOkg21f3eJ6u1Hmcv2csqTKh4ArgLWBObbPkPQ54Hjg0y3O77ukyX9mxarJUBrX5+vn79j+30P8e0RExCoYjXdcJ0taQGl4tgBWB7aSdK6kAygrFrdzef29KzDL9m9tvwxcAryzzTl32X7Q9nLgUlZMHv4DKxqgtpOHge83DY+fysoToI9q2t+y0UpyRkRE53S04aqDHfYD9rC9E3AvsAZlsvAs4CTggn6KaAxtV5vyd28aKHFg3dxu8vBLXjGEsuVk5JGS5IyIiM7p9KPC9YGnbD8naVvgbZQ5WKvZvlLSA5RVj6FpUnALc4EvSdqY8qjwSOBc23MpeYLAqw3lbpK2pDwqPJwyZ2vMJDkjImJkdbrhugk4UdJC4H7K48LJwKymQROn198zgPPq+6M9mgux/bik04FbKb2vG2xf0+aadwKfBXakpHDM7O8Ga09tmu0zB1mn5ndcS23vN8jzIiJiBIz7CchDUXtc021/YIxv5VWZgBwRMXRdMQE5IiJiMLq64aqLSC5ufLc9q5O9LUmfkjS9U+VHRMTAxlNyRk9Kcsb4kbSMiN7Q1T2uapKkr9Vkjptr6sbWkm6qiR3fryMaG7mD59VtP5HUsndWMwnPkfSDmtSxW9Pu7er+ByWdPCo1jIiIV/VCw7UN8BXb2wNPAx+mDIH/K9u7ANMpS580TAH2Bt5PGcX4Blpb2/bbgb8Evt60fVtgf0pCxyclvb7viZmAHBHROb3wqPAh2/Pr50YixtuBb9XYKCiTnhu+WRed/KmkBykN0XxWdimA7dmS1pO0Qd1+ve0XgRcl/QbYFHik+UTb51Pnj62x2Ta9M2wzImIc6IWG68Wmz8spDcnTNa6plZWSNSRdCLwVeMz2+9od1+Z6vfA3jIjoGr34j+7vgIckHWr7WyrdrrfYXlD3HyrpImBLYCvgftt/3qKcw4FbJe0FLLO9rKkHN2hJzoiIGFm92HABHAV8VdLfA68HLgMaDdf9wG2UntmJtl9oU8ZTkn4ArAcc1+H7jYiIQerqhsv2w8AOTd+/0LT7gDan3WH71EEUf6Xt05s32P5Un+87EBERo6oXRhVGRMQEMq4aLknH1kUjO+UQ21cMcA9rAY8CF9Y5XLdLWqfuW16XUFks6dtNIw0jImKUjLdHhccCi4HH+u6QNKkuDtlpfw382vaO9bpvBl6q+55vjFasAzxOAs7qr7AkZ4y9JGZE9JZR6XFJurqmWCypk3Mn1RSLxZIWSTpV0iHANMqyIfNrAsbDks6UdDtlNOCR9fjFks5uc619JM2WNFPSfTUpY7Wm/WdJWiBpjqRNWxSxGaXHBYDt++u8rb7upCzREhERo2i0HhUeV1MspgEnUxZ/nGx7h9qzubA+wpsHHGV7qu3Gmlcv2N6LsrbW2cC76vm7Sjq4zfV2A/6GsibX1sCH6va1gTl1NebZwPEtzv068LeS7pT0aUnb9D1A0iTg3cC1rS6e5IyIiM4ZrYbrZEkLKAtJbgGsDmwl6VxJB1DmXrVzef29KzDL9m9tvwxcAryzzTl32X6wPlq8FNirbv8DcF393EjZeI2awrEV8HlgI+CHkv5r3b2mpPnAE3Xfd1pd3Pb5tqfZnjZprfX7qVpERAxVxxuuurjjfsAetadzLyWCaSdgFuU90QX9FPH7RlFtyt+9PlqcX1czhvapFy95xcqZbVMvbD9r+yrbfwl8A2ikaTTecf0xpfE9qZ/7joiIDhiNwRnrA0/Zfq6mtL8N2BhYzfaVkh4AZtRjnwHWbVPOXOBLkjYGngKOBM61PZfy6BB4taHcTdKWwM8pCRjnD/ZmJe0J3Gf7KUmrA9tRGthX1RSNk4FrJH3V9kstigKSnBERMdJG41HhTcDrJC0E/j/K48LJwKz62G0G0JjoO4OS2D5f0prNhdh+vB53KyUF4x7b17S55p3AZykjFB8CZvZ3g5IOlPR/6tetgdskLaL0DucBV/Y9x/a99T6O6K/siIgYWVrx5Kw31B7X9E6uhDwU06ZN87x588b6NiIiuoqku21Pa7VvXE1AjoiIGMi4aLhGMjHD9qy+vS1Jzw7yPnarc8Dul/RjSRfUJA0kHSxpYd2+qJ+h+BER0UHjJTnjWMY4MaNORv4WcITtO+tyKB8G1q1zub4AvMf2Q3Xgx3ckPWh7YX/lJjlj5CUJI2Ji62iPq8sSM04CLrJ9J4CLK2z/GpgO/P+2H6r7HgI+A/zvEf2DRUTEgDr9qLCbEjN2oExKbmX7Fvvm1e0rSXJGRETndLrh6prEjAGIlSc1t9oGJDkjIqKTOtZwdWFixhJglzb3soTSa2y2M3Bfu5uPiIjO6OTgjK5KzAD+CbhL0vW1bCT9GfBdysCMb0n6nu2HJU0B/g44ZKBCk5wRETGyOtlw3QScWBMz7ue1iRmNnl7fxIzngT2aC7H9uKRGYoaAGwaRmLEj5V3WgIkZwDTbZ9r+taQjgC9I+k/AK7WMq2z/StLfAt+W9HrK+lwfr4G8ERExinomOWO8JWY0JDkjImLokpwRERE9o5carqnAYQMdJOmURhpGi30zJD0nad2mbV+S5PqObdApHBER0RnjJTljJJxCWTvruVU87mfAQcA36ru4fYFHh3tTSc4YvCRiRMRgdGWPS9Lakq6vSRiLJX0S2By4VdKt9Ziv1knASyT9Q912ct/jWriUMiIRYB/gDuDlTtYnIiIGrysbLuAA4DHbO9neATiHknO4r+196zFn1Bd7bwH2lvQW219ucVxfPwU2kbQhZej9ZUO9uSRnRER0Trc2XIuA/SSdLekdtlu1DodJuocy8Xl7ykrGg3UVZYHI3YHvD/XmkpwREdE5XfmOy/ZPJO0CvA/4jKSbm/fXScjTgV1tPyVpBvCGIVziMuAeSujuKyUoPiIixoOubLjq2l1P2v5GHeV3LCvSN5YC61Eio5bVJPj3UmKm6HNcS7Z/IekMSmrGKklyRkTEyOrKhouSjPF5Sa9QUiw+SkncuFHS47b3lXQvJWPwQcoAi4bz+xx3AXCe7dfMErb9L22uvZakR5q+f9H2F0eoXhERMYCeSc4Yr5KcERExdEnOiIiInpGGaxAkfbCmZ2zbtG2KpMVjeV8RERNRt77jGm1HArdThsh/aigndmtyRlIsImK8mlA9LklHS1pYEzculnRoTd5YIGl2m3PWAfYE/oLScEVExBiaMD0uSdsDZwB72l4qaSPgNmB/249K2qDNqQcDN9W5Y09K2tn2PQNc6wTgBIBJ620yYnWIiIiJ1eN6F3CF7aUAtp+kDJOfIel4YFKb85pjny6r3/uV5IyIiM6ZMD0uyurJrxn7b/tESbsD7wfmS5pq+4lXT5DeSGnwdpBkSuNmSR8f7EUzATkiYmRNpB7XLZT8wjcCSNpI0ta259o+k5KksUWfcw4B/s32H9ueYnsL4CFgr1G984iIeNWEabhsLwHOAm6TtAD4IiV9Y1Ed1j4bWCBpc0k31NOOBGb2KepK4COjdd8REfFaSc7osCRnREQMXZIzIiKiZ/TE4AxJpwDn235uuMfVpU8OAza1/Uzd9iXgZGCTOoR+OWUtsIbLbH+2v2t20wTkTDqOiG7QKz2uU4C1RuC4nwEHAUhaDdgXeLRp//O2pzb99NtoRUTEyOu6hkvS2pKur2kXiyV9EtgcuFXSrfWYr0qaJ2mJpH+o207ue1wLlwKH18/7UOZ5vdzJ+kRExNB0XcMFHAA8Znsn2zsA5wCPAfva3rcec0Z9qfcWYG9Jb7H95RbH9fVTYBNJG/LaiccNa0qa3/Rz+MpFlOSM2nDOW/7cslWqbEREvFY3NlyLgP0knS3pHbZbtQyHSboHuBfYHthuCOVfRckk3B34fp99fR8VXt6qgCRnRER0TtcNzqiZgbsA7wM+I+nm5v2StgSmA7vafqoOunjDEC5xGXAPcJHtVySt0v0mOSMiYmR1XY9L0ubAc7a/AXwB2Bl4Bli3HrIe8HtgmaRNgfc2nd58XEu2f0EJ4/3nEb71iIgYAV3X4wJ2pCRevAK8BHwU2AO4UdLjtveVdC+wBHiQMsCi4fw+x10AnGf7NTOEbf9Lm2uvKWl+0/ebbJ82MtWKiIjBSHJGhyU5IyJi6JKcERERPWPYjwolPQxMa6xvNcwypgFH2z55uGWMpuHUeTwnZyQpIyK60Zi+46rvlvIcLSIiBm3AR4WSpkj6saSLJC2UdIWkRmzSX0m6py4Nsq2k1ST9VNIm9dzVJP1M0saSDq1JFwskza7795F0Xf28jqQLa1kLJX1Y0iRJM+p5iySd2uL+9m6aEHyvpHVVfL7pvMObjv943bZA0kqRTQPUd6U6D+3PHRERq2qw77jeTAmnfQvwO+Av6/altncGvgpMt/0K8A3gqLp/P2BBfbR2JrC/7Z2AA1tc4xPAMts71ut8D5gKTLa9g+0dgQtbnDcdOMn2VOAdwPPAh+q5O9V7+LykzSS9FzgY2L3ex+eGWN+V6tzq5CRnRER0zmAbrl/abgwr/wYrVgC+qv6+G5hSP38dOLp+Po4Vjc0dwAxJxwOTWlxjP+ArjS+2n6IMZ99K0rmSDqA0In3dAXyxZhFuYPvlen+X2l5u+9fAbcCu9RoXNtLhbT85xPq2q/NrJDkjIqJzBvuOq++Y+cb3F+vv5Y2ybP9S0q8lvYsSm3RU3X6ipN2B9wPzJU3tU6b6XqcmX+wE7A+cRFl25Lg+x3xW0vWUJI05kvarZbWy0jUkbQF8u349D7ipn/q2rHN/kpwRETGyBtvj+s+S9qifjwRuH+D4Cyg9lW/aXg4gaWvbc22fCSwFtuhzzs3AxxpfJG0oaWNgNdtXUh4l7tz3QrXcRbbPpgz02BaYDRxe35FtArwTuKte47jGOytJG9n+ZVP24HnDrG9ERIySwTZcPwKOkbQQ2Ijyfqc/1wLr8Np3Up+vAxoWUxqWBX3O+TSwYWMAB2UtrMnArJpWMQM4HUDSiZJOrOed0nTO88CNwExgYb3G94CP2/6V7Zvqvc2rZbZ8RzWM+kZExCgZMDlD0hTgurqEyOAKLfOz/tH2O1bt9kbfcOrbnyRnREQMXX/JGSM+j0vSaZT8wKMGOjYiImKoklXYgqQ3URao3JUyGONh4FPA5cAetn9Vj/tn4Be2V5oP1rDGZtt4s2PO6ewND0JSMiKimySrcAhUFuCaCcyyvbXt7YC/A9YAzqYspYKknSnD5P/vWN1rRMRE1I3LmgyJpKMpgzBMGbBxLfBJynD2Zbbf2eeUfYGXmkYYYnt+LesOyqCNfYGzgI/ZfqnjlYiIiFf1dMMlaXvKopB72l4qaSPKZOT9bT8qaYMWp+1AmVy8kroi8kcpIxWvtT27zXVPAE4AmLTeJqtekYiIeFWvPyp8F3BFI829JmUMlODRr9r7Wkw/KyQnOSMionN6usdF6zSOlRI8bD/RdMgS4JAByn2l/gwoyRkRESOr13tctwCHSXojlKSMQSR4fA9Yo/bIqOftKmnvUbvriIhoq6d7XLaXSDoLuE3ScuBeYD1J21B6Y7cACyRtDlxg+322LemDwDl1TtoLlOHwp4xJJSIi4jUyj6vDkpwRETF0mcc1SJKW1wUpF0v6dvOoQ0nbS/qepJ+oLJb5iTrnKyIiRlF6XE0kPWt7nfr5IuAnts+StCZlJOFHbd9c0+WvpGQafqWfIsc0OSNpGRHRrSZkj0vS0ZIWSlog6WJJhzZS5CW1nH/Vx52UdHqAjwB32L4ZoC5E+THgtM7cfUREtNOTgzOGOfG4+fxJwLuBf62btqfPpGTbD0haR9J6tlutzBwRER3Qqz2u4U48XrOu0/UEZR2u79TtK80Ha7LSdkknSJonad7y55YNvxYREbGSXm24Wk48Bv6eMm9rfmNuVx/P254K/DGwOnBS3b4EeM2zVklbAc/afqZvIUnOiIjonJ4cnFEfFc6kLEHyRH1UuKHtB+r+e4E/b4TnNp3XPDjjrcA1wNaUR6pLgBNsf7cO1vgW8B+2z+3vXjIcPiJi6EZ1IcnxYDgTj1uUca+kBcARti+WdBBwrqSvUB41Xgz802jVKSIiip7scY0n6XFFRAzdhBwOHxERvaknGi5JUyQt7kC5x9bHiY3vD0vaeKSvExERg9eT77hG0LGUxIzHhlvAokeXMeW060fshoYiyRkR0Yt6osdVTZL0NUlLJN0saU1JW0u6SdLdkr4vaVsASTMknVe3/UTSB/oWJukQyhD4S2p+4Zp1119JukfSokZ5ERExenqp4doG+Irt7YGngQ8D5wN/ZXsXYDqvXbV4CrA3ZUHJ8yS9obkw21cA84CjbE+1/XzdtdT2zsBXa5kryQTkiIjO6aVHhQ81zcu6m9IwvR34VlOI+xpNx3/T9ivATyU9CGwLzGdgVzVd40OtDrB9PqXRZI3NtsmwzYiIEdRLDdeLTZ+XA5sCT9ckjFb6NiiWdCHwVuCxVnO7+lxnOb3194uI6Aq9/A/v74CHJB1q+1t17ay32F5Q9x9aly7ZEtgKuN/2n/cp4xlg3VW5iR0nr8+8DJKIiBgxvfSOq5WjgL+oCRhLgIOa9t1PSYy/ETjR9gstzp9Bef/VPDgjIiLG0IRMzpA0g7II5BWdvlaSMyIihi7JGRER0TPGXcPVN62iA+U/a/vY/npbkvaRdF3T/VjSu5v2f7BuO6RT9xkREa2Nx8EZx9ImrULSJNvLR/2OYBFwJCVVHuAIYEH7w5tOTHJGRMSIGrUel6Sra4LFkjpBd1JNsFhcUyhObZVWUfMBz5R0O2Uk4JH1+MWSzm5zrX0kzZY0U9J9NSVjtab9Z0laIGmOpE0HcfvfB3aT9HpJ6wD/hcHN+YqIiBE2mj2u42w/WUfn/ZAygXey7R0AJG1g+2lJHwOm255XtwO8YHuv+ghxDrAL8BRws6SDbV/d4nq7AdsBPwduokwWvgJYG5hj+wxJnwOOBz49wL0b+C6wP7A+cC1lGH1Lkk4ATgCYtN4mAxQdERFDMZrvuE6uw9LnAFsAqwNbSTpX0gGUeVftXF5/7wrMsv1b2y8DlwDvbHPOXbYfrI8WLwX2qtv/AFxXPzcSNgbjMsojwiNqeW3ZPt/2NNvTJq21/iCLj4iIwRiVhkvSPsB+wB62d6KsSLwGsBMwCzgJuKCfIn7fKKpN+bvXR4vzJR1YN6+UjFF/v+QVcwAGnX5h+y5gB2Bj2z8ZzDkRETHyRutR4frAU7afq4nqbwM2BlazfaWkByiTfaH/tIq5wJfqmlhPUQZMnGt7LjC1cVBtKHeTtCXlUeHh1OzAVXQ60GqicltJzoiIGFmj1XDdBJwoaSElsWIOMBmY1TRo4vT6ewYlreJ5YI/mQmw/Lul04FZK7+sG29e0ueadwGeBHYHZwMz+brD21KbZPrPdMbZv7K+MiIjovJ5Mzqg9rum2V1pna7QlOSMiYuiSnBERET1jPE5AXmW2Z1EGfURERI/pyYZrqCS9rg6vH3FjlZyR1IyI6FUTouGS9AnKEie/BJZS5m99APgBsCdlQvH/bTp+S+DfKX+fm4BTba9T930c+O/AK8CNtk8bvZpERETPN1ySpgEfpqxs/DrgHkrDBbCB7b1bnPYl4Ku2/03SSU1lvRc4GNi9Du3fqM01k5wREdEhE2Fwxl7ANbaft/0M8O2mfZe3OWdPVqRjXNy0fT/gQtvPAdh+stXJSc6IiOicidBwtUzbqH4Pr4buzpc0v2lfq3kCarM9IiJGSc8/KgRuB/5F0mco9X0/8LXmA2yfAZzRtOkOSibhNyjvxhpuBs6U9O+NR4Xtel0NSc6IiBhZPd/jsv1DyuCLBcBVwDxg2QCn/TVwkqQfUuKqGmXdVMuaV3tn0ztxzxER0V5PJmf0JWkd289KWosS/3SC7XuGcP6zjVGFQ5XkjIiIoesvOWMiPCoEOF/SdsAbgIuG0mhFRMT4MiEaLtsfGe65ks4Afl4Dgl8B/qftuZJm0bTgZUREjI4J0XANl6Q9KBOVd7b9Yl1OZfWhlJHkjIiIkTWhGi5JR1MGVBhYSBlo8UnKgpLLbPddTXkzYKntFwFsLx3F242IiBYmTMMlaXvKkPc9bS+tqRe3AfvbflTSBi1Oawx//wnwXeBy27cN4lpJzoiI6JCeHw7f5F3AFY1eU51/dQcwQ9LxwKS+J9h+FtiF0gj9Frhc0rEDXSjJGRERnTORGq6VUi9snwj8PbAFMF/SG/ueZHu57Vm2Pwl8jJJ7GBERY2TCPCoEbgFmSvpH20/UR4Ub2p4LzJX0p5QG7InGCZLeDLxi+6d101Tg50O5aJIzIiJG1oRpuGwvkXQWcJuk5cC9wHqStqH0xm4BFkjaHLjA9vuAdYBz6/uvl4GfUd9dRUTE2JgQyRljKckZERFD119yxkR6xxURET0gDVcfkt4k6TJJD0i6T9INkv5E0hRJz9flT+6T9G+SXj/W9xsRMdFMmHdcgyFJwExKnuERddtUYFPgl8ADtqdKmgR8BzgMuKS/MkcrOSNJGRExUfR0wzWMpIx9gZdsn9fYYHt+LWtK07blku4CJnfy/iMiYmU923ANMyljB+DuQZT9BmB3yrpdrfYnOSMiokN6+R3XkJMyBmHruoDkE8AvbC9sdVCSMyIiOqdne1y0ScqQtDvwfkpSxlTbTzQdsgQ4pJ8yG++4NgNmSTrQ9rX93UQmIEdEjKxe7nHdAhzWiHGStJGkrW3PtX0msJSSlNHse8AatUdGPW9XSXs3H2T7ceA04PSO1iAiIlbSsw2X7SVAIyljAfBF4POSFklaDMymJmVIuqGeY+CDwHvqcPglwKeAx1pc4mpgLUnv6HxtIiKiIckZHZbkjIiIoUtyRkRE9IxeHpwxZJKetb1O0/djgWm2P1a/Hw18nDLwQ8DXbX+hvzI7OQE5k44jYiJKj2uQJL0XOAX4b7a3B3YGlo3pTUVETEA923BJOlrSQkkLJF0s6VBJi+v32cMo8nRguu3HAGy/YPtrI3vXERExkJ58VDjM1AyANesE44aNKDFRMMhUjXr9JGdERHRIr/a4hpua8bztqY0f4MzhXDzJGRERndOTPS6Gl5oxkCXALpRJyoOW5IyIiJHVqz2u4aRmDOQzwOckvamWuYakk0f0riMiYkA92eOyvURSIzVjOXAvsJ6kbSi9sVuoqRnABbbfN4gyb5C0KfDdum6Xga93rhYREdFKkjM6TNIzwP1jfR8dsDGl59pLerFOkHp1k16sEwyvXn9su+Xotp7scY0z97eLLelmkub1Wr16sU6QenWTXqwTjHy9evUdV0RE9Kg0XBER0VXScHXe+WN9Ax3Si/XqxTpB6tVNerFOMML1yuCMiIjoKulxRUREV0nDFRERXSUNV4dIOkDS/ZJ+Jum0sb6foZC0haRbJf1I0hJJf123byTpO5J+Wn9v2HTO6bWu90vaf+zuvn+SJkm6V9J19Xsv1GkDSVdI+nH9b7ZHj9Tr1Pr/32JJl0p6QzfWS9LXJf1G0uKmbUOuh6RdJC2q+75cgxDGTJt6fb7+f7hQ0szmQPMRrZft/IzwDyXE9wFgK2B1YAGw3Vjf1xDufzNg5/p5XeAnwHbA54DT6vbTgLPr5+1qHdcAtqx1nzTW9WhTt/8F/DtwXf3eC3W6CPgf9fPqwAbdXi9gMvAQsGb9/k3g2G6sF/BOyvp9i5u2DbkewF3AHpT0nxuB947Dev034HX189mdqld6XJ2xG/Az2w/a/gNwGXDQGN/ToNl+3PY99fMzwI8o/5AcRPlHkvr74Pr5IOAy2y/afgj4GeVvMK5I+iNKyPIFTZu7vU7rUf4B+VcA23+w/TRdXq/qdZSlhl4HrAU8RhfWy/Zs4Mk+m4dUD0mbAevZvtPlX/t/azpnTLSql+2bbb9cv84B/qh+HtF6peHqjMnAL5u+P1K3dR1JU4C3AnOBTW0/DqVxA/5TPaxb6nsO8HHglaZt3V6nrYDfAhfWR6AXSFqbLq+X7UeBLwC/AB4Hltm+mS6vV5Oh1mNy/dx3+3h2HKUHBSNcrzRcndHqGW3XzTuQtA5wJXCK7d/1d2iLbeOqvpI+APzG9qAWA6UL6lS9jvK45qu23wr8nvLoqZ2uqFd953MQ5bHS5sDakv6sv1NabBt39RqEdvXoqvpJOgN4GbiksanFYcOuVxquzniE1y6b8keUxxxdQ9LrKY3WJbavqpt/Xbv21N+/qdu7ob57AgdKepjy6PZdkr5Bd9cJyn0+Yntu/X4FpSHr9nrtBzxk+7e2XwKuAt5O99erYaj1eIQVj92at487ko4BPgAcVR//wQjXKw1XZ/wQ2EbSlpJWB44Arh3jexq0OqrnX4Ef2f5i065rgWPq52OAa5q2H6GyRtmWwDaUF67jhu3Tbf+R7SmU/x7fs/1ndHGdAGz/CvilpDfXTe8G7qPL60V5RPg2SWvV/x/fTXnX2u31ahhSPerjxGckva3+PY5uOmfckHQA8LfAgbafa9o1svUay1EpvfwDvI8yGu8B4Iyxvp8h3vtelO76QmB+/Xkf8EbKWmY/rb83ajrnjFrX+xnj0U6DqN8+rBhV2PV1AqYC8+p/r6uBDXukXv8A/BhYDFxMGZHWdfUCLqW8p3uJ0sP4i+HUA5hW/xYPAP9ETT4aZ/X6GeVdVuPfjfM6Ua9EPkVERFfJo8KIiOgqabgiIqKrpOGKiIiukoYrIiK6ShquiIjoKmm4IiKiq6ThioiIrvL/AEcJnvPwH5rXAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "subject_counts = df['arxiv_subjects'].apply(lambda x: x[0]).value_counts()\n",
    "subject_counts[subject_counts>80].sort_values().plot(kind = 'barh')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a8f29954",
   "metadata": {},
   "source": [
    "The plot above is a bar plot of how many papers have a subject as their primary subject on arXiv -- in the full dataset -- with at least 80 occurences in the dataset.\n",
    "Perhaps not suprisingly, we find booming sections in computer science leading, along with astrophysics.\n",
    "\n",
    "Out of curiousity, let's check secondary sections:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "45f9e231",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:>"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAckAAAD4CAYAAACHbh3NAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAxGklEQVR4nO3de7ydRX3v8c+XIJFwFbEIaA3EVMpFIiRBhCJQPFBUxAsC4oGIJaXVUjmlLZQWsadUUI8F0ZpGKkGliISbIlIUCREkgQC5oqBctASqhEvkfgnf88fMJoudtXb23llr752V7/v1ymuv9Twz88zsaIaZZ+Y3sk1ERESsar3hrkBERMRIlU4yIiKihXSSERERLaSTjIiIaCGdZERERAvrD3cFon223HJLjx07drirERGxVrntttuW2X5ds3vpJLvI2LFjmTdv3nBXIyJirSLpV63uZbp1mEl6ssX1j0paKGmJpAWSzpO0+RBXLyJinZaR5Agk6SDgROBPbC+VNAo4BtgKeLxVvkVLlzP25O8PTSUjIkaI+898d8fKTifZRpKOBk4CDCwEvgt8GlgBLLe9Tz+LOhU4yfZSANsrgK+3v8YREdGXdJJtImknSue2l+1lkrYAbgAOrKPBzQdQ3E7A7f187lRgKsCoTZu+d46IiEHKO8n22R+YaXsZgO1HgZuAGZKOA0YNplBJu0iaL+keSYf3vm97uu2JtieOGrPZmtQ/IiJ6yUiyfUSZZn2Z7eMl7QG8G5gvaYLtR/pR1hJgN+B624uACZK+DGzYV6Zdtt2MeR2cm4+IWNdkJNk+1wEflvRaAElbSBpne67t04BlwBv7WdZngS9IekPDtT47yIiIaL+MJNvE9hJJZwA3SFoB3AFsKmk8ZZR5HbBA0jbAebYPrlnHSHqgoagv2v6ipNcBP6grWx8HFgP/NVTtiYgIUM6T7B4TJ050gglERAyMpNtsT2x2L9OtgyDpU5LGrEk6STMkPS1pk4Zr50iypC3r96aBBiIiYmhkunVwPgV8C3h6DdP9Engf8C1J6wH7AUsHW6kEE4iIZjq52b7bZSS5GpI2kvT9GhpusaRPA9sA10u6vqb5qqR5NYTcZ+q1E3qna+IioGdbx76ULSMvdrI9ERHRf+kkV+8g4EHbu9reGTgbeBDYz/Z+Nc2pdT77rcA7Jb3V9peapOvtF8DrJL0GOBL4dicbEhERA5NOcvUWAQdIOkvSH9le3iTNhyXdTlnRuhOw4wDKvww4AtgD+MlAKydpah3FzlvxdLOqRUTEYOWd5GrYvlvS7sDBwGclXdt4X9J2lHitk2w/JmkG8OoBPOLblBB0F9h+SdJA6zcdmA4weuvxWaocEdFG6SRXo+5rfNT2t+pq0ynAE8AmlAABmwJPAcslbQX8CTCrZm9M15TtX0s6FfjRmtY1EXciItorneTq7QJ8XtJLwAvAnwN7Ujb6P2R7P0l3UELJ3UtZfNNjeq905wHTbL9iM6Ptf2/x7KaBBtrUroiIWI0EE+giCSYQETFwCSYQERExCOkkRxhJ769Rd3ZouDZW0uLhrFdExLoo7yRHniOBGynbQk4fSMZE3IkYmESiidXJSLJDJB0taWGN1PNNSYfViD0LJM1ukWdjYC/g45ROMiIihlFGkh0gaSfgVGAv28skbQHcABxoe6mkzVtkPRS4pu7NfFTSbrZvX82zpgJTAUZt+rq2tSEiIjKS7JT9gZm2lwHYfpSyNWSGpOOAUS3yNYam+3b93ifb021PtD1x1JjN1rzmERHxsowkO0PAK/bW2D5e0h7Au4H5kibYfuTlDNJrKZ3rzpJM6Ugt6W/7+9AEE4iIaK+MJDvjOko819cCSNpC0jjbc22fRonA88ZeeT4EfMP2m2yPtf1G4D5g7yGteUREvCydZAfYXgKcAdwgaQHwRUrUnkV1K8dsYIGkbSRdXbMdCVzeq6hLgY8MVb0jIuKVEnGniyTiTkTEwCXiTi+SptTA5Z0q/8l+ppssabakuyT9XNJ5ksbUe4fWLSQ/ryPQQztV34iIaG5dXbgzBVhMORT5FSSNsr2i0xWoJ4ZcAhxh+2aVM7I+CGwiaTzwBeBdtu+rx3H9UNK9the2KjPBBIZeNqNHdLeuGklKukLSbZKW1MOIR0maUTfxL5J0oqQPAROBCyXNl7ShpPslnSbpRuAwSUf2vD+UdFaLZ+1bR4GXS7pT0jRJ6zXcP6MGDphTO8TePkE5Q/JmABczbf+Gcj7lv9i+r967D/gs8Ddt/YVFRESfuqqTBI61vTulEzwBmABsa3tn27sA59ueCcwDjrI9wfYzNe+ztvemLKo5i7IdYwIwqY+pzsnAX1OO0xoHfKBe3wiYY3vXWt5xTfLuDNzWotydmtybV6+/Qv2PgXmS5q14enmL4iIiYjC6rZM8oa4mnUPZYrEBsL2kcyUdBPyuj7wX15+TgFm2H7b9InAhsE+LPLfYvrdOz17Eyu0azwNX1c+3AWMH2I5V9lm2uJZgAhERHdQ1naSkfYEDgD3rCO4OYDSwKzCLMr15Xh9FPNVTVIvy96jTs/MlHVIv9+60er6/4JXLhlfQ/N3vEmD3FnVZQhkNN9oNuLNV5SMiov26aeHOZsBjtp+ux0y9HdgSWM/2pZLuAWbUtE8Am7QoZy5wjqQtgcco+xfPtT2XMv0KvNwpT66Lan4FHA5MH0B9vwzcIun7tWwkfRT4EWXRziWSfmz7fkljgb+nBBxoKRF3IiLaq5s6yWuA4yUtBO6iTLluC8xqWFBzSv05A5gm6Rlgz8ZCbD8k6RTgesqo8mrbV7Z45s3AmZR3krNZNRjAK9QR6ETbp9n+jaQjgC9I+j3gpVrGZbb/R9LfAd+T9CrgBeBvbc/v5+8iIiLaIMEEBqmOJE+y/Z5hrsrLEkwgImLgEkwgIiJiENJJDpLtWWs6ipT0eknflnRP3Wt5taQ/kHSfpLf0Snv2QE4EiYiINZfp1mFSI+z8lBJQYFq9NoGyoOhgyr7Nz9Tr6wG/phzi/KtWZY7eery3PubsDtd83ZOoOhHdra/p1m5auDOsJB1NiZRjYCHwXeDTlC0gy2333mu5H2WryLSeCz0LcyQtp+zb/Ey9tQ9wf18dZEREtF86yTaQtBNwKmWkt0zSFsANwIG2l0ravEm2lhF3bC+U9JKkXW0vAI6gBCto9uypwFSAUZu+bs0bExERL8s7yfbYH5hpexmA7UeBm4AZko4DRg2izIuAIyStD7yPEgx9FYm4ExHRORlJtscqIeNsHy9pD+DdwHxJE2w/0pBkCX0HB7gIuJYyIl1o+7erq0SCCUREtFdGku1xHfBhSa8FkLSFpHG259o+DVhGiSXb6MfA6DrSpOabJOmdALbvAR6hBCtoOtUaERGdlU6yDWwvAc4AbqgB1r8IfL7nuC1KJJ0FkraRdHXNY+D9wLvqFpAlwOm88ozLi4AdWE0kn4iI6IxsAekiibgTETFwibjThKQpkrbpYPlP9iPNGEkXNhzwfKOkjeu9FfXEkcWSvtdihWxERHTQurxwZwqwmFdObwIgaVQ9I7LT/gr4TT0Qmhpl54V67xnbE+r1CyhHfZ3RV2GLli5n7Mnf71xt10IJBBARa6LrRpKSrpB0m6QlkqZKGiVpRh2RLZJ0oqQPUc5rvLCO1jaUdL+k0yTdCBwm6ciGEd5ZLZ61r6TZki6vYeWmNZw4gqQzJC2QNEfSVk2K2BpY2vPF9l22n2uS7mbKiSYRETGEuq6TBI61vTulEzyBcgbktrZ3riO2823PBOYBR9meYPuZmvdZ23tTFtqcRdn/OAGYJOnQFs+bDPw15bisccAH6vWNgDn1AOjZwHFN8n4d+DtJN0v6Z0njeyeQNAr4Y0oEn1XU/xCYJ2neiqeXt/ylRETEwHVjJ3lCXWE6h7LtYgNge0nnSjoI+F0feS+uPycBs2w/bPtF4EJKaLhmbrF9b52evQjYu15/Hriqfr4NGNs7Yw1Dtz3weWAL4FZJf1hvbyhpPmUbyBbAD5s9PMEEIiI6p6s6yXrG4wHAnnUEdwcwGtgVmEV5r3deH0U81VNUi/L3qNOz8+sBytAriEDD9xe8cunwClq8/7X9pO3LbP8F8C1KcHNY+U7yTZSO/hN91DsiIjqg2xbubAY8ZvtpSTsAbwe2BNazfamke4AZNe0TlBM3mpkLnCNpS+Ax4EjgXNtzKdOvwMud8mRJ2wG/Ag4Hpve3spL2Au60/ZikDYAdKZ35y2wvl3QCcKWkr9p+oUlRQCLuRES0W1eNJIFrgPUlLQT+L2XKdVtgVp26nAGcUtPOAKb1LNxpLMT2QzXd9cAC4HbbV7Z45s2UqDiLgftYzcZ/SYdI+qf6dRwlAMEiyqh3HnBp7zy276j1OKKvsiMior0STGAN1JHkSWt6+HK7JJhARMTAJZhARETEIGQk2U+SxgJX2d55iJ53OvCk7S/0N8/orcd762PO7lidRqIEC4iINZWRZERExCCkkxyYUZK+VqP5XFsj9YyTdE2N8vOTuqqWGuVnWr12t6Sm7y0lzZJ0tqSf1ug+kxtu71jv31tXuEZExBBKJzkw44Gv2N4JeBz4IGXLx1/WKD8nAf/WkH4s8E7KwcvTJL26Rbkb2X4H8BeUKDw9dgAOpET1+bSkV/XOmIg7ERGd0237JDvtvholB1ZG0XkHcIn0cvyB0Q3pv2P7JeAXku6ldHrzWdVFALZnS9q04cSP79dYrs9J+i2wFfBAY0bb06l7M0dvPT4vmCMi2iid5MA0Bh9fQem0Hu85raOJVaLxSDofeBvwoO2DW6Vr8bw+/74STCAior0y3bpmfgfcJ+kwABW7Ntw/TNJ6ksZRYrTeZftjNaj6wQ3pDq/59waW2868aUTECJCR5Jo7CviqpH8AXgV8mxIdB+Au4AbKiPN428+2KOMxST8FNgWO7XB9IyKin9JJ9pPt+4GdG7437l88qEW2m2yf2I/iL7V9SuMF26f3+j4k+zMjImKldXK6VdIUSdt0sPwn+5luMiVg+mWSbpf0fUm79EqzQNJFHahmRESsxro6kpxCCUj+YO8bkkbVsyHXiO0pfd2XtBXwHeBg2z+t1/amBD1fVL//IeU/ZPaRtJHtp1qVB7Bo6XLGnvz9Na36WiGRdiJiKHTVSFLSFXVT/5K6f3BU3dS/WNIiSSdK+hAwEbiw5wQQSfdLOk3SjZTFNkfW9IslndXiWftKmi3pckl31sAB6zXcP6OOAufUDrG3TwIX9HSQALZvtH1FQ5qPAN8ErgUOISIihlRXdZLAsXVT/0TgBMpU5ra2d7a9C3C+7ZmUI6mOqqtMn6l5n7W9NzAbOAvYv+afJOnQFs+bDPw1sAtlBPiBen0jYE49+Hk2cFyTvDsBt6+mPYcDF1P2UR7ZLEGCCUREdE63dZInSFpAOUfyjcAGwPaSzpV0EGXLRisX15+TgFm2H7b9InAhsE+LPLfYvrdOz14E7F2vPw9cVT/3BB3ok6S5kn4m6Zz6fRLwsO1fAdcBu0l6Te98tqfbnmh74qgxm63uMRERMQBd00nWsx0PAPasI7g7KNFvdgVmAZ8AzuujiJ73fWp2U9IedXp2vqSeqc9WQQBe8MrjVVoFAVgC7PZyRnsP4B+Bnp7uSGAHSfcD91C2h3ywj/pHRESbddPCnc2Ax2w/XYOMvx3YEljP9qWS7gFm1LRPAJu0KGcucI6kLYHHKJ3VubbnUqZfgZc75cmStgN+RZkanT6A+n4FmCvpvxreS46pZa8HHAa81fbSem0/4B/oo6NPxJ2IiPbqpk7yGuB4SQspm/jnANsCsxoW1PTsRZxBCTj+DLBnYyG2H5J0CnA9ZVR5te0rWzzzZuBMyjvJ2cDlfVWwjkAn2j7N9v9IOhw4S9K2wG+BZcA/UaZ3l/Z0kNVsyqkgW9t+aDW/i4iIaIMcujxIdSR5ku2mR2ANh4kTJ3revHnDXY2IiLVKDl2OiIgYhIwku8jorcd762POHu5qdESCB0REp2QkOYJI6qb3wBERXS3/YLeZpH+knAzy35SFOLcB7wF+CuwFfBf4fw3ptwP+k/J3cQ1wou2N672/Bf438BLwA9snD11LIiIinWQbSZpI2cv4Nsrv9nZKJwmwue13Nsl2DvBV29+Q9ImGsv4EOBTYo25r2aLFM6cCUwFGbfq6djUlIiLIdGu77Q1cafsZ208A32u4d3GLPHtRovVAidPa4wBKGL2nAWw/2ixzIu5ERHRORpLt1TRaT/UUlMDnwLsBbE+o95qtnlKL6y0lmEBERHtlJNleNwLvlfRqSRtTO8NGtk+tgdUn1Es3AUfUz0c1JL0WOFZSTxSeptOtERHROekk28j2rZSFOQuAyyinjazuaI6/Aj4h6VZWxm3F9jW1rHmS5gMndaLOERHRWvZJtpmkjW0/WUeAs4Gptld3JFZj/id7VrcOVCLuREQMXPZJ9iJpiqRtOlT8dEkrKCtbL23VQdZDm6/qdW0GMKp+niXproaTR2Z2qL4REdHCurpwZwqwGHiw9w1Jo+r5kINi+yOSDrG9wyCL+N8Nn4+y3e+h4aKlyxl78vcH+diRJRF2ImIk6KqRpKQrJN0maYmkqZJGSZohabGkRZJOlPQhYCJwYR2hbSjpfkmnSboROEzSkTX9YklntXjWvpJmS7pc0p2SpjWcNoKkMyQtkDRH0lZD9CuIiIg26raR5LG2H5W0IXArZSP/trZ3BpC0ue3HJX2ScoLHvHod4Fnbe9dp2DnA7pTzJK+VdKjtK5o8bzKwI+U8yWuADwAzgY2AObZPlfQ54Djgn5vk/6O6KKfH7wONU7AX1uO8AH5o+296F5BgAhERndNVI0ngBEkLKJ3cG4ENgO0lnSvpIOB3feTt2ew/CZhl+2HbLwIXUs53bOYW2/fW6dmLKMEEAJ5nZWd3GzC2Rf6f9GwHqVtCvtvr/lEN91fpICHBBCIiOqlrOsl6vuMBwJ62dwXuAEYDuwKzgE8A5/VRxFM9RbUof4+GRTSH1Mu9lwb3fH/BK5cNr6D7RuwREeuEbvrHezPgsRrndAfg7cCWwHq2L5V0DzCjpn0C2KRFOXOBcyRtSZluPRI41/ZcYEJPotopT64Byn8FHA5Mb3ObBiQRdyIi2qubOslrgOMlLQTuoky5bgvMalhQc0r9OQOYVt/37dlYiO2HJJ0CXE8ZVV5t+8oWz7wZOBPYhbIn8vK+KlhHoBNtn9bPNjW+k1xm+4B+5ouIiDZIMIFBqiPJk2y/Z5ir8rIEE4iIGLgEE2gTSWMlLR7uekRExNDopunWIWV7FmVB0IixtgYTSOCAiBipMpIcuFGSvlYDFlxbgxGMk3RNDWTwk7pwiBrIYFq9drekplOzzfJLWl/SrXVaF0mfrcdsRUTEEMlIcuDGA0faPk7Sd4APAh8Djrf9C0l7AP8G7F/TjwXeCYwDrpf0ZtvP9ipzeu/8tveXNAWYKekE4CBgj043LiIiVkonOXD32Z5fP/cECngHcEmN3ANlf2aP79h+CfiFpHuBHYCe/NRzJ5vmt71E0jeB71H2fz7fuzKJuBMR0TnpJAfuuYbPK4CtgMcbDlHubZWAA5LOB95GCbB+xGry7wI8Xp+zauH2dOr+zNFbj89S5YiINkonueZ+B9wn6TDbl6gMB99qe0G9f5ikC4DtgO2Bu2x/rLEASU3zS/oA8FpKWLyrJE22/XiriiSYQEREe2XhTnscBXy8xo1dAryv4d5dwA3ADyjvHXu/j2yav0b8ORP4uO27gS8D53SwDRER0UuCCXRQPUT5KttDcmBygglERAxcgglEREQMQt5JdpDtKatLI+n1wNmUI7qeA+4HPkU9bqvnLMyIiBh66SSHUV2kczlwge0j6rUJlJWs/z3Q8tbGiDuJthMRI1k6yTaSdDRwEmXbx0LKIcqfpmwVWW679+HN+1HOnpzWc6FnD6aksUNQ5YiI6EM6yTaRtBNwKrCX7WWStqCsaj3Q9lJJmzfJtjMlIMGaPDfBBCIiOiQLd9pnf2Cm7WUAth8FbgJmSDoOGNWJh9qebnui7YmjxmzWiUdERKyzMpJsH9Eruo7t42ss1ncD8yVNsP1IQ5IlwIfaVYEEE4iIaK+MJNvnOuDDkl4LIGkLSeNsz7V9GrAMeGOvPD8GRteRJjXfJEnvHLJaR0RESxlJtkkNRn4GcIOkFcAdwKaSxlNGmdcBCyRtA5xn+2DblvR+4GxJJwPPsnILCMBbJD3Q8JgTbV8yRE2KiFjnpZNsI9sXABesJtmDwMENeR4EPtwi7avaVLWIiBiETLcOgqSxkhZ3oNwpdaTZ8/3+GsM1IiKGQUaSI8sUYDFltDlga1swgQQSiIiRLiPJwRsl6WuSlki6VtKGksZJukbSbZJ+ImkHKIHOJU2r1+6W9J7ehUn6EDARuFDSfEkb1lt/Kel2SYt6youIiKGRTnLwxgNfsb0T5VDkD1IOP/5L27tTIu/8W0P6scA7KdtBpkl6dWNh9aSQecBRtifYfqbeWmZ7N+CrtcyIiBgimW4dvPt6QshRouaMBd4BXFJCsgIwuiH9d2y/BPxC0r3ADsB8Vu+yhmd8oPfNRNyJiOicdJKD91zD5xWUoOSP257QIn3vgzst6XzgbcCDtg9ukqfxOSto8vdlezplBMvorcfncNCIiDZKJ9k+vwPuk3SY7UvqCR9vtb2g3j9M0gXAdsD2wF22P9arjCeATQZbgUTciYhor7yTbK+jgI9LWkAJOfe+hnt3UQKe/wA43vazTfLPoLyvbFy4ExERw0R2Zug6TdIMygHKMzv5nIkTJ3revHmdfERERNeRdJvtic3uZSQZERHRwjrfSfaOctOB8p+0PaWvUaSkfSVd1VAfS/rjhvvvr9fadmJIRESsXhbu9BHlRtIo2yuGvEawCDiSEhQd4AhgQevkNVMi7kREtFXXjiQlXVEj3yyRNFXSqBr5ZnGNXnNisyg3NV7qaZJupKxIPbKmXyzprBbP2lfSbEmXS7qzRtdZr+H+GZIWSJojaat+VP8nwGRJr5K0MfBm+renMiIi2qibR5LH2n60rhK9lbIZf1vbOwNI2tz245I+CZxke169DvCs7b3rNOwcYHfgMeBaSYfavqLJ8yYDOwK/Aq6hbPyfCWwEzLF9qqTPAccB/7yauhv4EXAgsBnwXcrWkVUkmEBEROd07UgSOKFuxZhDOex4A2B7SedKOoiyr7GVi+vPScAs2w/bfhG4ENinRZ5bbN9bp2cvAvau158HrqqfeyLz9Me3KdOsR9TymrI93fZE2xNHjdmsn0VHRER/dOVIUtK+wAHAnrafljSLEiJuV8ro7BOUMxyPbVHEUz1FtSh/D+Df69fTKB3uKhF16s8XvHKfTdOoOc3YvkXSzsAztu9uCHXXUoIJRES0V1d2kpQpysdqB7kD8HZgS2A925dKuoeycR/6jnIzFzinnun4GGUxzbm25wITehLVTnmypO0o062HU0PFraFTgGZBByIiYgh0ayd5DXC8pIWUSDdzgG2BWQ0Lak6pP2dQotw8A+zZWIjthySdAlxPGVVebfvKFs+8GTgT2AWYDVzeVwUlHQJMtH1aqzS2f9BXGRER0VmJuNMGdSR5ku1VzokcSom4ExExcIm4M4wkfUrSmAHmGStpcafqFBER/dOt061DyvYsYFaL258CvgU83el6rE3BBBJIICLWBuv8SFLSqZLukvQjSRdJOknSLEkT6/0tJd1fP4+V9BNJt9c/76jX9615Zkr6uaQLVZwAbANcL+n6Js+eIulKSdfUOny64fYoSV+rwRCuzakgERFDb53uJCXtTtmH+DbK5v9Jq8nyW+BdtnejrGD9UsO9t1FGjTtSzovcy/aXKOHu9rO9X4syJ1OO2JpAifDTMy8+HviK7Z2Ax4EPtmjDVEnzJM1b8fTy1VQ/IiIGYp3uJIE/Ai63/bTt31Ei2/TlVcDXJC0CLqF0iD1usf2A7ZcoIeTG9rMOP7T9iO1ngMtYGYTgPtvz6+eWQQgSTCAionPW9U4SVg0CAPAiK383r264fiLwG0pQgomUKD49nmv43DRoQD3NY3790zNibBWEYLXlRUREZ63r//DOBmZIOpPyu3gvJZLO/ZR4rbcAjcdTbQY8YPslSccAo/rxjJ5gBctsX07D/skaUeddkrYAngEOpXUUoNVKxJ2IiPZap0eStm+nxGmdD1xKOX0D4AvAn0v6KSVST49/A46RNAf4A1aGr+vLdOAHzRbuVDcC3+ypQ0+g9YiIGH4JJtBA0unAk7a/METPm0KJuvPJdpSXYAIREQOXYAIRERGDkE6yge3Th2oUWZ1PwwKdukfz9Pr5dElLGxb6zJe0+RDWLSJinbeuL9wZbs8BH5D0WdvLmtz/14F02mtLxJ1E24mItUVGkm0i6WhJCyUtkPRNSYdJWly/z26R7UXKwp4Th7CqERHRTxlJtoGknYBTKVF2ltUtHTcAB9peuppp0q8ACyV9rsm9EyV9tH5+rFnUHklTgakAozZ93Zo0IyIieslIsj32B2b2TJnafhS4ibIH8zj62E9ZI/18Azihye1/tT2h/mka1i4RdyIiOicjyfYQvSLn2D5e0h7Au4H5kibYfqRF/rOB2ykLeQYtwQQiItorI8n2uA74sKTXAkjaQtI423NtnwYsA97YKnMdeX4H+PiQ1DYiIvolI8k2sL1E0hnADZJWAHcAm0oaTxllXgcskLQNcJ7tg5sU8/+A3kEFGt9JAhxq+/72tyAiIppJxJ0ukog7EREDl4g7I5ik10v6tqR7JN0p6WpJkyXdJ+n1Den+TdLJw1nXiIh1TaZbh5EkUU4FucD2EfXaBMqpIWdRAq1/VNJulHMmd++rvJESTCDBAiKiW6STbCNJRwMnUVa6LqQc4vxpynmQy23v0yvLfsALtqf1XOg5aFnSTZQTR/YDzgA+afuFjjciIiJelk6yTQYZUGBn4LZm5dUzK/8c+DHwXdtNo/YkmEBEROfknWT7DDqgQCt1VLmYco5lqzQJJhAR0SHpJNunaUAB4B8oeyTn9+yjbLCE1bxnBF6qfyIiYohlurV9rgMul/Svth+p062vsT0XmCvpvZTOsjHqzo+Bf5F0nO2vAUiaBIyxfcNAK5CIOxER7ZVOsk0GE1DAtiW9Hzi7bu94Frgf+NSwNCIiIl4hwQS6SIIJREQMXIIJREREDEKmW0cYSacCH6HsrXwJ+DPbcyXNAk6y3XKoOJzBBBJAICK6UTrJEUTSnsB7gN1sPydpS2CDYa5WRMQ6K51kBw0iAs/WwDLbzwH07LmMiIjhkU6yQwYZgeda4DRJdwM/Ai5e3VaQRNyJiOicLNzpnAFH4LH9JCW4wFTgYeBiSVP6ekgi7kREdE5Gkp3TNAKPpD2Ad1Mi8Eyw/UivNCuAWcAsSYuAY4AZ/XlggglERLRXRpKdcx3w4Z5QdJK2kDTO9lzbpwHLKBF4XibpLTX4QI8JwK+GqsIREfFKGUl2yGAi8AAbA+fW95UvAr+kvm+MiIihl4g7XSQRdyIiBm7ERdyRNEPShzpQ7r6S3jGA9JtL+ot2pWsnSVMkfXkonxkREa/UbdOt+wJPAj/tZ/rNgb+gj/MaB5huWA1HxJ1E2omIbtavkaSkoyUtlLRA0jclvUnSdfXadZJ+v6abIelLkn4q6d6e0aKKL0u6U9L3gd9r8Zx9Jd0g6TuS7pZ0pqSjJN0iaZGkcTXdeyXNlXSHpB9J2krSWOB44ERJ8yX9Ua+yd6rlzK/1Hg+cCYyr1z4vaePantvr895Xs78iXZN6z5D0VUnX13a/U9LXJf1M0oyGdP9L0s21/EskbVyvT6q/swW1jpvULNtIukbSLyR9rj9/VxER0T6rHUm22BR/AfAN2xdIOhb4EnBozbI1sDewAyXCzEzg/cBbgF2ArYA7ga+3eOSuwB8CjwL3Uha1TJb0V8BfUo6RuhF4ez1q6k+Bv7X915KmAU/a/kKTco8HzrF9oaQNKPsUTwZ2tj2htnV94P22f1dDws2R9N3e6Vp4DWVv5CHA94C9gD8FbpU0AXiAcgDzAbafkvR3wP+RdCZwMXC47VslbQo8U8ucALwNeA64S9K5tv+78aEJJhAR0Tn9mW5dZVO8SozRD9T73wQaRzlX2H4JuFPSVvXaPsBFdQ/gg5J+3MfzbrX9EICkeyhRaAAWAfvVz2+gbLTfmhLb9L5+tONm4FRJbwAus/0LSb3TiHII8j6U4OLbUjr1/vhe7bQXAb+xvai2YQkwttZ5R+Cm+twNap3eAjxk+1YA27+r+QCus728fr8TeBPwik7S9nRgOsDorcdnFVZERBv1Z7p1lU3xTTTef65X3mZpyk1pjzqFOV/SIU3yv9Tw/SVWdurnAl+2vQvwZ8CrV1M/bP8nZZT3DPBfkvZvkuwo4HXA7nXU+JtmZUs6o6feDZcb69m7DetTfhc/tD2h/tnR9sfp+/fbWM4Kuu8dckTEiNaff3SvAy6X9K+2H6nTrT8FjqCMIo+iTH/2ZTbwZ5K+QXkfuR/wn7bnUqYUgfJOsp/13gxYWj8f03D9CWDTZhkkbQ/ca/tL9fNbgQXAJg3JNgN+a/sFSftRRm495b6czvaplCnogZgDfEXSm23/UtIYyujy55R3j5PqdOsmrJxuHZBE3ImIaK/VdpItNsWfAHxd0t9QYox+bDXFXE6Ztl0E3E0J9L0mTgcukbSU0vlsV69/D5hZF9z8JeU94cQa4eZw4KOSXgD+B/inOnV8k6TFwA+As4DvSZoHzKd0YNT/OHg5ne2/GWiFbT+sEof1Ikmj6+V/sH23pMMpQQQ2pHSQBwy0/IiIaL8EE+giCSYQETFwGmnBBCIiItYGWQgyjCS9HjgbmERZpHM/ZYvL88DPgLsoq2DnAR+3/UJf5SWYQEREe2UkOUxU9nhcDsyyPc72jsDfs3LLyT11he0ulAU+Hx6WikZErMMykmwTSUcDJ1G2cyykBFL4NGXrxnLb+/TKsh/wgu1pPRdsz69ljW24tkLSLZQ9mxERMYTSSbZBi6hENwAH2l6qcvRVbzsDt/Wj7FcDewB/1eJ+Iu5ERHRIplvbY5WoRMBNwAxJx1FC4A3UuBqs4BHg17YXNktke7rtibYnjhqz2eBqHxERTWUk2R6rRM2xfbykPYB3A/MlTbD9SEOSJUBfx4XdY3tCDb03S9Ihtr/bVyUSTCAior0ykmyP64APS3otgKQtJI2zPbcGMlgGvLFXnh8Do+tIk5pvkqR3NiaqcWxPBk7paAsiImIV6STbwPYSoCcq0QLgi8Dn63Fbiylh+RZI2kbS1TWPKaejvEvSPTUQ+unAg00ecQUwRr2O/4qIiM5KxJ0ukog7EREDl4g7LUiaImmbDpb/ZD/S7CtpecNpKPMlHVDvrajfF9dDmsd0qq4REbGqdX3hzhRgMU2mOCWNqudfDoWf2H5Pk+vPNBwIfSHl4OgvtiokEXciItqrK0eSkq6QdJukJZKmSholaUYdkS2SdKKkDwETgQvraG1DSfdLOk3SjcBhko7sea8o6awWz9pX0mxJl0u6U9I0Ses13D9D0gJJc7TyEOrB+Anw5jXIHxERA9SVnSRwrO3dKZ3gCZQzK7e1vXM9qPl82zMpMVGPqocg95zh+KztvSmLbc6i7IGcAEySdGiL500G/poSQm4c8IF6fSNgju1da3nHNc/OH/Wabh3XeFPS+sCfUI4ao9e9qZLmSZq34unlff9WIiJiQLq1kzyhrjKdQ9l6sQGwvaRzJR0E/K6PvBfXn5MocVUftv0icCHQO7Rcj1ts31unZy8C9q7Xnweuqp9vA8a2yP+T2lH3/LmnXt+wBhSYB/wa+I/eGRNMICKic7runaSkfSmHFu9p+2lJs4DRwK7AgcAnKMHCj21RxFM9RbUofw/g3+vX0ygdbu8lwj3fX/DK5cMrGPjv++V3khERMfS6rpMENgMeqx3kDsDbgS2B9WxfKukeYEZN+wSwSYty5gLnSNoSeAw4EjjX9lzK9Cvwcqc8WdJ2wK+Aw4HpbW5TvyTiTkREe3VjJ3kNcLykhZTzGOdQTtCY1bCgpid6zQxgmqRngD0bC7H9kKRTgOspo8qrbV/Z4pk3A2dS3knOphyB1ZKkQ4CJNRoP1HeSDUn+ub4zjYiIYZRgAmuojiRParGFY0glmEBExMAlmEBnvRl4+YwqSadLOml1mSRtIOnsGpLuF5KulPSGhvuvl/Ttev9OSVdL+oMOtSEiIproxunWofYicOsg8v0L5X3oH9SDlT8GXFYXBkGZsr3A9hEAkiYAWwF3typwqIMJJJBARHS7dJKApLGUd5k3Uhb6LADOBz4D/B5wVE16NrAh8AzwMeA+4J8oWzX2Bj5b0+1YV9X+PnC27S/1et6Ymn+7nqg+ts+XdCxlX6YpK2On9eSxPb+dbY6IiNXLdOtKbwbOAd4K7AB8hLLf8STg74GfA/vYfhtl68e/2H6+fr647m/s2WO5A2W7yWTg05Je1eRZv7bde7/mPGAnYGfKvsqIiBhGGUmudJ/tRQD12KrrbFvSIkoQgM2ACySNp4z0end8jb5v+zngOUm/pUyTPtBwf5VDmntdb7pHsxlJU4GpAKM2fd1qUkdExEBkJLnScw2fX2r4/hLlPyb+L3C97Z2B9wKv7mdZzYII/BJ4k6TeezR3A+4ElgC796fSibgTEdE5GUn232bA0vp5SsP1vgISNGX7KUkXAF+UdHxduHM0MAb4cU32L5KOs/01AEmTgDG2b2hVboIJRES0V0aS/fc54LOSbgJGNVy/nrJQZ76kw/sqoG7j6Dm/8hTgWeBuSb8ADgPe7wp4P/CuugVkCXA6TY70ioiIzkkwgS6SYAIREQOXYAIRERGDkE5ymEhaUadoF0v6nqTNG+7tJOnHku6u0Xj+UVK/V7xGRER7ZLp1mEh60vbG9fMFwN22z5C0IbAY+HPb19bAA5cCV9n+Sl9ljt56vLc+5uxOVx1ItJ2I6B6Zbu0wSUdLWihpgaRvSjqsjhAXSJrdjyJuppxUAiWIwU22rwWw/TTwSeDkztQ+IiJayRaQNSRpJ+BUYC/byyRtAdwAHGh7aeM0aov8o4A/Bv6jXtqJXtF2bN8jaWNJm/aO0pNgAhERnZOR5JrbH5hpexmA7UeBm4AZko7jldtFGm1Yz5B8BNgC+GG93ioaD82uJ5hARETnZCS55lbp1GwfX0/zeDcwX9IE24/0yveM7QmSNgOuAj4BfIkSbWefVzxA2h540vYTfVUkwQQiItorI8k1dx3wYUmvBZC0haRxtufaPg1YBryxVWbby4ETgJNqIPQLgb0lHVDL25DSeX6uw+2IiIheMpJcQ7aXSDoDuEHSCuAOYNMaCF2UTnRBjbRznu2Dm5Rxh6QFwBG2vynpfcC5kr5Cma79JvDloWpTREQU2QLSRSQ9Adw13PVoky0po/C1Xbe0A9KWkahb2gHD25Y32W668jEjye5yV6u9PmsbSfO6oS3d0g5IW0aibmkHjNy25J1kREREC+kkIyIiWkgn2V2mD3cF2qhb2tIt7YC0ZSTqlnbACG1LFu5ERES0kJFkREREC+kkIyIiWkgn2SUkHSTpLkm/lDSiTwyR9EZJ10v6maQlkv6qXt9C0g/rGZo/lPSahjyn1LbdJenA4av9qiSNknSHpKvq97W1HZtLminp5/XvZs+1uC0n1v9tLZZ0kaRXry1tkfR1Sb+VtLjh2oDrLml3SYvqvS8N9Zm0Ldrx+fq/r4WSLtcrz9Edke3Adv6s5X8oUXnuAbYHNgAWADsOd736qO/WwG718ybA3cCOlNB7J9frJwNn1c871jaNBrarbR013O1oaM//Af6TcuYna3E7LgD+tH7eANh8bWwL5di5+4AN6/fvAFPWlrZQYjfvBixuuDbgugO3AHtSIn/9APiTEdCO/wWsXz+ftTa0IyPJ7jAZ+KXte20/D3wbeN8w16kl2w/Zvr1+fgL4GeUftvdR/qGm/jy0fn4f8G3bz9m+D/glpc3DTtIbKIHsz2u4vDa2Y1PKP2r/AWD7eduPsxa2pVqfctLO+sAY4EHWkrbYng082uvygOouaWtgU9s3u/Q032jIMySatcP2tbZfrF/nAG+on0dsO9JJdodtgf9u+P4AKw9xHtEkjQXeBswFtrL9EJSOFPi9mmwkt+9s4G+BlxqurY3t2B54GDi/Th2fJ2kj1sK22F4KfAH4NfAQsNzlEPO1ri0NBlr3bevn3tdHkmMpI0MYwe1IJ9kdms3Rj/i9PZI2Bi4FPuVeh0n3Ttrk2rC3T9J7gN/avm21iWuWJteGvR3V+pSpsa/afhvwFGVar5UR25b6vu59lGm7bYCNJH20ryxNro2ItvRDq7qP6DZJOhV4kXLqEYzgdqST7A4P8MrjuN5AmV4aseqxYJcCF9q+rF7+TZ1eof78bb0+Utu3F3CIpPspU9z7S/oWa187oNTtAdtz6/eZlE5zbWzLAcB9th+2/QJwGfAO1s629Bho3R9g5VRm4/VhJ+kY4D3AUXUKFUZwO9JJdodbgfGStpO0AXAE8N1hrlNLdXXafwA/s/3FhlvfBY6pn48Brmy4foSk0ZK2A8ZTXuYPK9un2H6D7bGU3/mPbX+UtawdALb/B/hvSW+pl/4YuJO1sC2Uada3SxpT/7f2x5T33mtjW3oMqO51SvYJSW+vv4OjG/IMG0kHAX8HHGL76YZbI7cdQ7lKKH869wc4mLJK9B7g1OGuz2rqujdlymQhML/+ORh4LeX8zV/Un1s05Dm1tu0uhnh1Wz/btC8rV7eule0AJgDz6t/LFcBr1uK2fAb4ObCYch7r6LWlLcBFlHepL1BGUh8fTN2BibX991DOo9UIaMcvKe8ee/5/P22ktyNh6SIiIlrIdGtEREQL6SQjIiJaSCcZERHRQjrJiIiIFtJJRkREtJBOMiIiooV0khERES38f0q1nvpwGKrIAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# get df where length of arxiv_subjects is 2 or more\n",
    "df_subjects_multiple = df[df['arxiv_subjects'].apply(lambda x: len(x)>1)]\n",
    "list_secondary = df_subjects_multiple['arxiv_subjects'].apply(lambda x: x[1:]).to_list()\n",
    "list_secondary_flat = [item for sublist in list_secondary for item in sublist] # flatten list\n",
    "# return to dataframe and plot\n",
    "secondary_df = pd.Series(list_secondary_flat).value_counts()\n",
    "secondary_df[secondary_df>100].sort_values().plot(kind = 'barh')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1d5c0564",
   "metadata": {},
   "source": [
    "This can be compared with https://arxiv.org/help/stats/2021_by_area\n",
    "\n",
    "Some cute sociological trends can be found. For example, even though the math section of arXiv is a factor of ~2 less prolific than cs, the math section is highly under-represented in our dataset, perhaps indicating that the math community is considerably less engaged in Twitter compared to math and physics communities."
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "cfe51a626cda2b044dfd7223b65bea869adf91f1a20dfea9a2d4979b930fc724"
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}