tweet_cleaning.ipynb

{
 "metadata": {
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3-final"
  },
  "orig_nbformat": 2,
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3.8.3 32-bit ('.venv')",
   "metadata": {
    "interpreter": {
     "hash": "3046998e069a8ee93e30444cc403e4617040b9c68fe86dacb97e5dfa0b33401a"
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2,
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Importing, chaning to personal directory\n",
    "import pandas as pd \n",
    "import matplotlib.pyplot as plt \n",
    "import os\n",
    "\n",
    "os.chdir('C:\\\\Users\\\\micha.DESKTOP-U2HVTMF\\\\MyPythonScripts\\\\Jupyter\\\\Datasets')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "                id_str        user_name  \\\n0  1362854450390577153  MarkoSilberhand   \n1  1362854453406294016       maxasteele   \n2  1362854459928289280    ggspillthetea   \n3  1362854464114331649  THEkarliehustle   \n4  1362854464512794631    mikeymikedoha   \n\n                                                text             hashtags  \\\n0  RT @MarkoSilberhand: @lesleyabravanel This is ...  TexasFreeze,TedCruz   \n1  Big congrats all around for maximizing profits...                  NaN   \n2  @kodeerants it unfortunately is. https://t.co/...                  NaN   \n3  I just donated to @TRAEABN and his #ReliefGang...           ReliefGang   \n4  RT @Reuters: Citigroup lets Texas staff shelte...                  NaN   \n\n               created  user_followers  user_friends  user_favorites  \\\n0  2021-02-19T19:59:56           33205         32414           83426   \n1  2021-02-19T19:59:56           14016          5858           28322   \n2  2021-02-19T19:59:58            6459           206           73698   \n3  2021-02-19T19:59:59           25130           750           98331   \n4  2021-02-19T19:59:59            7837          7814            7645   \n\n                                        expanded_url  \\\n0                                                NaN   \n1                                                NaN   \n2  https://www.wfaa.com/article/money/business/ri...   \n3  https://twitter.com/i/web/status/1362854464114...   \n4                             http://reut.rs/3awKrvg   \n\n                                    user_description         user_created  \\\n0  Dissatisfied with the situation of the world. ...  2017-08-17T17:15:35   \n1  Senior Comms Advisor, @American_Bridge. Alum o...  2012-03-09T00:28:23   \n2     she/her; welcome to me yelling into the ✨void✨  2018-10-16T22:12:53   \n3  first a music fan. also an xennial documentary...  2010-06-11T06:34:30   \n4  Sonder – n. the profound realisation that each...  2009-03-07T09:21:30   \n\n         user_location              source usr_mentions  \n0  Bayern, Deutschland     Twitter Web App          NaN  \n1       Washington, DC           TweetDeck          NaN  \n2     Hot Mess Express  Twitter for iPhone          NaN  \n3         Brooklyn, NY     Twitter Web App          NaN  \n4     Bristol, England  Twitter for iPhone          NaN  \n"
     ]
    }
   ],
   "source": [
    "#Creating df from csv, collected from Kaggle https://www.kaggle.com/rajsengo/texas-winter-strom-2021-tweets/\n",
    "\n",
    "tweets_df = pd.read_csv('tweets_txwx2021.csv')\n",
    "print(tweets_df.head())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "208\n"
     ]
    }
   ],
   "source": [
    "#Counting matches of word within column given dataframe\n",
    "def countMatches(df, column, word):\n",
    "    '''Looks within given dataframe and column for word, counts the number of matches'''\n",
    "    count = 0\n",
    "    for string in df[column]:\n",
    "        if word in string:\n",
    "            count += 1\n",
    "    \n",
    "    return count\n",
    "\n",
    "countMatches(tweets_df, 'text', 'wind')\n",
    "print(count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "                   id_str       user_name  \\\n48    1362854571786178562     raythemouse   \n108   1362854753361887234    AngelinaDDD2   \n111   1362854765240193024     suziqkelley   \n245   1362855244166742020  StevieStacks84   \n1311  1362859025986162691   thinkingmanDJ   \n\n                                                   text hashtags  \\\n48    'This is a disaster': Texas faces dwindling fo...      NaN   \n108   @clifford6111 Yuck. It has been so yucky here ...      NaN   \n111   RT @BryanM_KVUE: The problem is that the natur...      NaN   \n245   RT @catturd2: The worthless windmills froze in...      NaN   \n1311  RT @catturd2: The worthless windmills froze in...      NaN   \n\n                  created  user_followers  user_friends  user_favorites  \\\n48    2021-02-19T20:00:25           11468         11319           32412   \n108   2021-02-19T20:01:08           17012          1513           54093   \n111   2021-02-19T20:01:11            5023          5420          133880   \n245   2021-02-19T20:03:05           23134           305          153752   \n1311  2021-02-19T20:18:07           16256         14439            6086   \n\n                                           expanded_url  \\\n48    https://www.theguardian.com/us-news/2021/feb/1...   \n108                                                 NaN   \n111                                                 NaN   \n245                                                 NaN   \n1311                                                NaN   \n\n                                       user_description         user_created  \\\n48    #Dogs \\n#Cats\\n#Nature\\n#Deforestation \\n#Envi...  2013-06-20T22:16:08   \n108   Soccer mom, , dog lover, beach bunny, working ...  2017-05-19T21:43:02   \n111   Love kids, animals, family, friends, ghost sto...  2012-03-13T03:35:08   \n245                              #TeamStacks #AsYouWish  2010-12-02T00:35:46   \n1311                                           Think...  2008-12-11T18:17:06   \n\n                  user_location               source usr_mentions  \n48      Perth Western Australia      Twitter Web App          NaN  \n108   District of Columbia, USA      Twitter Web App          NaN  \n111                       Tejas  Twitter for Android          NaN  \n245                     Mars 👽🛸   Twitter for iPhone          NaN  \n1311                        NaN   Twitter for iPhone          NaN  \n"
     ]
    }
   ],
   "source": [
    "#Filtering, creating dataframe with tweets containing certain words\n",
    "\n",
    "def find_word(df, column, word):\n",
    "    '''Looks within the given dataframe and column for rows that contain the given word and returns a new dataframe with only those tweets'''\n",
    "    #TODO: make it so that it can take multiple words, smth with *args\n",
    "\n",
    "    return_df = pd.DataFrame()\n",
    "\n",
    "    for string in df[column]:\n",
    "        if word in string:\n",
    "            return_df = return_df.append(df.loc[df[column] == string])\n",
    "\n",
    "    return return_df\n",
    "\n",
    "\n",
    "wind_df = find_word(tweets_df, 'text', 'wind')\n",
    "\n",
    "print(wind_df.head())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "                    id_str        user_name  \\\n12812  1363322557118570496   allinwithchris   \n2313   1362862496067952643   KurtSchlichter   \n15938  1363330627991343104  TrueFactsStated   \n17823  1363336464503816194   PsychicHealerC   \n2118   1362861852619726848      nomadicmatt   \n\n                                                    text       hashtags  \\\n12812  “Did AOC, 10 years ago at the tender age of 21...            NaN   \n2313   RT @ThePeoplesCube: BREAKING: Biden administra...  TexasBlackout   \n15938  RT @DrEricDing:Let this sink in—Congresswoman ...            NaN   \n17823  RT @DrEricDing:What sent Texas reeling is not ...            NaN   \n2118   We don’t have drinkable water in Texas but we ...            NaN   \n\n                   created  user_followers  user_friends  user_favorites  \\\n12812  2021-02-21T03:00:01          401228           846             794   \n2313   2021-02-19T20:31:54          291341          2319            1619   \n15938  2021-02-21T03:32:05          226652          4300          252411   \n17823  2021-02-21T03:55:17          142051        149845          337588   \n2118   2021-02-19T20:29:21          120502          1233             605   \n\n                                            expanded_url  \\\n12812  https://twitter.com/i/web/status/1363322557118...   \n2313                                                 NaN   \n15938                                                NaN   \n17823                                                NaN   \n2118                                                 NaN   \n\n                                        user_description         user_created  \\\n12812  @chrislhayes hosts the Emmy Award-winning “All...  2013-03-21T15:59:46   \n2313   Free speech absolutist | Loud + Proud Conserva...  2008-12-13T00:09:25   \n15938  Chair of Mad Dog Pac. Worked in Clinton White ...  2014-11-23T23:31:32   \n17823  Mom, #Life, Steeler's, Writer, Psychic LifeCoa...  2011-07-29T10:42:09   \n2118   NYT best selling author of How to Travel the W...  2008-10-04T06:27:54   \n\n              user_location               source    usr_mentions  \n12812                   NaN            TweetDeck             NaN  \n2313     Manhattan Beach CA      Twitter Web App             NaN  \n15938  DC and on the road.    Twitter for iPhone  DrEricDing,AOC  \n17823             Ohio, USA  Twitter for Android      DrEricDing  \n2118       Around the World   Twitter for iPhone             NaN  \n"
     ]
    }
   ],
   "source": [
    "#Sorting dataframe with tweets that mention wind by follower count, dropping duplicates\n",
    "\n",
    "wind_sorted = wind_df.sort_values('user_followers', ascending=False)\n",
    "\n",
    "wind_sorted_nodupes = wind_sorted.drop_duplicates() #BUG: should not need to drop duplicates but many are created: perhaps error when iterating, creates new row every time there's a match?\n",
    "\n",
    "print(wind_sorted_nodupes.head())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "#outputting to csv\n",
    "\n",
    "wind_sorted_nodupes.to_csv(r'C:\\Users\\micha.DESKTOP-U2HVTMF\\MyPythonScripts\\Jupyter\\DataOutputs\\texas_wind.csv', index=False)\n",
    "\n",
    "#TODO: Can implement NLP to automatically determine the sentiment of the top tweets. For now, have to do manually. Fails to scale but can see how the very top influencers are spreading information"
   ]
  },
  {
   "source": [
    "Conclusions:\n",
    "\n"
   ],
   "cell_type": "markdown",
   "metadata": {}
  }
 ]
}