impeach_tweets.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import tweepy as tw\n",
    "import pandas as pd\n",
    "from textblob import TextBlob\n",
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tweepy\n",
    "from tweepy.error import TweepError\n",
    "from tweepy.models import Status"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "tweets = []\n",
    "\n",
    "# Open connection to file\n",
    "with open(\"Impeach_tweets_noon.txt\", \"r\") as tweets_file:\n",
    "    # Read in tweets and store in list\n",
    "    for line in tweets_file:\n",
    "        tweet = json.loads(line)\n",
    "        tweets.append(tweet)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'created_at': 'Tue Feb 09 17:36:53 +0000 2021',\n",
       " 'id': 1359194573373902857,\n",
       " 'id_str': '1359194573373902857',\n",
       " 'text': 'https://t.co/usj6xbBti0',\n",
       " 'display_text_range': [0, 0],\n",
       " 'source': '<a href=\"https://mobile.twitter.com\" rel=\"nofollow\">Twitter Web App</a>',\n",
       " 'truncated': False,\n",
       " 'in_reply_to_status_id': None,\n",
       " 'in_reply_to_status_id_str': None,\n",
       " 'in_reply_to_user_id': None,\n",
       " 'in_reply_to_user_id_str': None,\n",
       " 'in_reply_to_screen_name': None,\n",
       " 'user': {'id': 6972272,\n",
       "  'id_str': '6972272',\n",
       "  'name': 'Brandon Rohwer',\n",
       "  'screen_name': 'brandonRohwer',\n",
       "  'location': 'FROM PARTS UNKNOWN (Brooklyn)',\n",
       "  'url': 'http://www.brandonrohwer.com',\n",
       "  'description': \"you can't spell #onbrand without brandon\",\n",
       "  'translator_type': 'none',\n",
       "  'protected': False,\n",
       "  'verified': False,\n",
       "  'followers_count': 1364,\n",
       "  'friends_count': 774,\n",
       "  'listed_count': 44,\n",
       "  'favourites_count': 25216,\n",
       "  'statuses_count': 52428,\n",
       "  'created_at': 'Wed Jun 20 17:05:27 +0000 2007',\n",
       "  'utc_offset': None,\n",
       "  'time_zone': None,\n",
       "  'geo_enabled': True,\n",
       "  'lang': None,\n",
       "  'contributors_enabled': False,\n",
       "  'is_translator': False,\n",
       "  'profile_background_color': '3D3B3D',\n",
       "  'profile_background_image_url': 'http://abs.twimg.com/images/themes/theme1/bg.png',\n",
       "  'profile_background_image_url_https': 'https://abs.twimg.com/images/themes/theme1/bg.png',\n",
       "  'profile_background_tile': True,\n",
       "  'profile_link_color': 'BD0000',\n",
       "  'profile_sidebar_border_color': '000000',\n",
       "  'profile_sidebar_fill_color': '969696',\n",
       "  'profile_text_color': '000000',\n",
       "  'profile_use_background_image': True,\n",
       "  'profile_image_url': 'http://pbs.twimg.com/profile_images/1146959579462459392/i97DkBve_normal.jpg',\n",
       "  'profile_image_url_https': 'https://pbs.twimg.com/profile_images/1146959579462459392/i97DkBve_normal.jpg',\n",
       "  'profile_banner_url': 'https://pbs.twimg.com/profile_banners/6972272/1564931277',\n",
       "  'default_profile': False,\n",
       "  'default_profile_image': False,\n",
       "  'following': None,\n",
       "  'follow_request_sent': None,\n",
       "  'notifications': None},\n",
       " 'geo': None,\n",
       " 'coordinates': None,\n",
       " 'place': None,\n",
       " 'contributors': None,\n",
       " 'quoted_status_id': 1359192915487064066,\n",
       " 'quoted_status_id_str': '1359192915487064066',\n",
       " 'quoted_status': {'created_at': 'Tue Feb 09 17:30:18 +0000 2021',\n",
       "  'id': 1359192915487064066,\n",
       "  'id_str': '1359192915487064066',\n",
       "  'text': 'Trump has just added another member to his legal team hours before the impeachment trial begins. @NBCNews @MSNBC',\n",
       "  'source': '<a href=\"https://mobile.twitter.com\" rel=\"nofollow\">Twitter Web App</a>',\n",
       "  'truncated': False,\n",
       "  'in_reply_to_status_id': None,\n",
       "  'in_reply_to_status_id_str': None,\n",
       "  'in_reply_to_user_id': None,\n",
       "  'in_reply_to_user_id_str': None,\n",
       "  'in_reply_to_screen_name': None,\n",
       "  'user': {'id': 32871086,\n",
       "   'id_str': '32871086',\n",
       "   'name': 'Kyle Griffin',\n",
       "   'screen_name': 'kylegriffin1',\n",
       "   'location': 'Manhattan, NY',\n",
       "   'url': 'https://www.instagram.com/griffinkyle/',\n",
       "   'description': \"Senior Producer. MSNBC's @TheLastWord. Opinions mine. Suburban women, will you please like me?\",\n",
       "   'translator_type': 'none',\n",
       "   'protected': False,\n",
       "   'verified': True,\n",
       "   'followers_count': 1127764,\n",
       "   'friends_count': 830,\n",
       "   'listed_count': 8710,\n",
       "   'favourites_count': 37700,\n",
       "   'statuses_count': 71019,\n",
       "   'created_at': 'Sat Apr 18 12:45:48 +0000 2009',\n",
       "   'utc_offset': None,\n",
       "   'time_zone': None,\n",
       "   'geo_enabled': False,\n",
       "   'lang': None,\n",
       "   'contributors_enabled': False,\n",
       "   'is_translator': False,\n",
       "   'profile_background_color': 'C0DEED',\n",
       "   'profile_background_image_url': 'http://abs.twimg.com/images/themes/theme1/bg.png',\n",
       "   'profile_background_image_url_https': 'https://abs.twimg.com/images/themes/theme1/bg.png',\n",
       "   'profile_background_tile': False,\n",
       "   'profile_link_color': '0084B4',\n",
       "   'profile_sidebar_border_color': 'C0DEED',\n",
       "   'profile_sidebar_fill_color': 'DDEEF6',\n",
       "   'profile_text_color': '333333',\n",
       "   'profile_use_background_image': True,\n",
       "   'profile_image_url': 'http://pbs.twimg.com/profile_images/961462238732275712/omWV1cZs_normal.jpg',\n",
       "   'profile_image_url_https': 'https://pbs.twimg.com/profile_images/961462238732275712/omWV1cZs_normal.jpg',\n",
       "   'profile_banner_url': 'https://pbs.twimg.com/profile_banners/32871086/1455243585',\n",
       "   'default_profile': False,\n",
       "   'default_profile_image': False,\n",
       "   'following': None,\n",
       "   'follow_request_sent': None,\n",
       "   'notifications': None},\n",
       "  'geo': None,\n",
       "  'coordinates': None,\n",
       "  'place': None,\n",
       "  'contributors': None,\n",
       "  'is_quote_status': False,\n",
       "  'quote_count': 34,\n",
       "  'reply_count': 154,\n",
       "  'retweet_count': 119,\n",
       "  'favorite_count': 521,\n",
       "  'entities': {'hashtags': [],\n",
       "   'urls': [],\n",
       "   'user_mentions': [{'screen_name': 'NBCNews',\n",
       "     'name': 'NBC News',\n",
       "     'id': 14173315,\n",
       "     'id_str': '14173315',\n",
       "     'indices': [97, 105]},\n",
       "    {'screen_name': 'MSNBC',\n",
       "     'name': 'MSNBC',\n",
       "     'id': 2836421,\n",
       "     'id_str': '2836421',\n",
       "     'indices': [106, 112]}],\n",
       "   'symbols': []},\n",
       "  'favorited': False,\n",
       "  'retweeted': False,\n",
       "  'filter_level': 'low',\n",
       "  'lang': 'en'},\n",
       " 'quoted_status_permalink': {'url': 'https://t.co/nGGhZooAvJ',\n",
       "  'expanded': 'https://twitter.com/kylegriffin1/status/1359192915487064066',\n",
       "  'display': 'twitter.com/kylegriffin1/s…'},\n",
       " 'is_quote_status': True,\n",
       " 'quote_count': 0,\n",
       " 'reply_count': 0,\n",
       " 'retweet_count': 0,\n",
       " 'favorite_count': 0,\n",
       " 'entities': {'hashtags': [],\n",
       "  'urls': [],\n",
       "  'user_mentions': [],\n",
       "  'symbols': [],\n",
       "  'media': [{'id': 1359194558630920193,\n",
       "    'id_str': '1359194558630920193',\n",
       "    'indices': [0, 23],\n",
       "    'media_url': 'http://pbs.twimg.com/media/EtzUP-_XAAEJsOm.jpg',\n",
       "    'media_url_https': 'https://pbs.twimg.com/media/EtzUP-_XAAEJsOm.jpg',\n",
       "    'url': 'https://t.co/usj6xbBti0',\n",
       "    'display_url': 'pic.twitter.com/usj6xbBti0',\n",
       "    'expanded_url': 'https://twitter.com/brandonRohwer/status/1359194573373902857/photo/1',\n",
       "    'type': 'photo',\n",
       "    'sizes': {'thumb': {'w': 150, 'h': 150, 'resize': 'crop'},\n",
       "     'medium': {'w': 1200, 'h': 675, 'resize': 'fit'},\n",
       "     'small': {'w': 680, 'h': 383, 'resize': 'fit'},\n",
       "     'large': {'w': 1200, 'h': 675, 'resize': 'fit'}}}]},\n",
       " 'extended_entities': {'media': [{'id': 1359194558630920193,\n",
       "    'id_str': '1359194558630920193',\n",
       "    'indices': [0, 23],\n",
       "    'media_url': 'http://pbs.twimg.com/media/EtzUP-_XAAEJsOm.jpg',\n",
       "    'media_url_https': 'https://pbs.twimg.com/media/EtzUP-_XAAEJsOm.jpg',\n",
       "    'url': 'https://t.co/usj6xbBti0',\n",
       "    'display_url': 'pic.twitter.com/usj6xbBti0',\n",
       "    'expanded_url': 'https://twitter.com/brandonRohwer/status/1359194573373902857/photo/1',\n",
       "    'type': 'photo',\n",
       "    'sizes': {'thumb': {'w': 150, 'h': 150, 'resize': 'crop'},\n",
       "     'medium': {'w': 1200, 'h': 675, 'resize': 'fit'},\n",
       "     'small': {'w': 680, 'h': 383, 'resize': 'fit'},\n",
       "     'large': {'w': 1200, 'h': 675, 'resize': 'fit'}}}]},\n",
       " 'favorited': False,\n",
       " 'retweeted': False,\n",
       " 'possibly_sensitive': False,\n",
       " 'filter_level': 'low',\n",
       " 'lang': 'und',\n",
       " 'timestamp_ms': '1612892213563'}"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tweet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>created_at</th>\n",
       "      <th>text</th>\n",
       "      <th>source</th>\n",
       "      <th>possibly_sensitive</th>\n",
       "      <th>lang</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Tue Feb 09 17:01:28 +0000 2021</td>\n",
       "      <td>RT @JesseBWatters: If you put Maxine Waters wo...</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>en</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Tue Feb 09 17:01:28 +0000 2021</td>\n",
       "      <td>Lindsey Graham Threatens to Call Democrats as ...</td>\n",
       "      <td>&lt;a href=\"https://mobile.twitter.com\" rel=\"nofo...</td>\n",
       "      <td>False</td>\n",
       "      <td>en</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Tue Feb 09 17:01:28 +0000 2021</td>\n",
       "      <td>@NunesAlt As long as it's not Trump streaking....</td>\n",
       "      <td>&lt;a href=\"https://mobile.twitter.com\" rel=\"nofo...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>en</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Tue Feb 09 17:01:28 +0000 2021</td>\n",
       "      <td>RT @chipfranklin: Who else thinks the vote to ...</td>\n",
       "      <td>&lt;a href=\"https://mobile.twitter.com\" rel=\"nofo...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>en</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Tue Feb 09 17:01:28 +0000 2021</td>\n",
       "      <td>RT @JaySekulow: HAPPENING NOW: Impeachment Tri...</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
       "      <td>False</td>\n",
       "      <td>en</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                       created_at  \\\n",
       "0  Tue Feb 09 17:01:28 +0000 2021   \n",
       "1  Tue Feb 09 17:01:28 +0000 2021   \n",
       "2  Tue Feb 09 17:01:28 +0000 2021   \n",
       "3  Tue Feb 09 17:01:28 +0000 2021   \n",
       "4  Tue Feb 09 17:01:28 +0000 2021   \n",
       "\n",
       "                                                text  \\\n",
       "0  RT @JesseBWatters: If you put Maxine Waters wo...   \n",
       "1  Lindsey Graham Threatens to Call Democrats as ...   \n",
       "2  @NunesAlt As long as it's not Trump streaking....   \n",
       "3  RT @chipfranklin: Who else thinks the vote to ...   \n",
       "4  RT @JaySekulow: HAPPENING NOW: Impeachment Tri...   \n",
       "\n",
       "                                              source possibly_sensitive lang  \n",
       "0  <a href=\"http://twitter.com/download/iphone\" r...                NaN   en  \n",
       "1  <a href=\"https://mobile.twitter.com\" rel=\"nofo...              False   en  \n",
       "2  <a href=\"https://mobile.twitter.com\" rel=\"nofo...                NaN   en  \n",
       "3  <a href=\"https://mobile.twitter.com\" rel=\"nofo...                NaN   en  \n",
       "4  <a href=\"http://twitter.com/download/iphone\" r...              False   en  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.DataFrame(tweets, columns=['created_at', 'text', 'source', 'possibly_sensitive', 'lang'])\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['created_at'] = pd.to_datetime(df.created_at)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['source'] = df['source'].str.extract('>(.+?)<', expand=False).str.strip() "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.rename(columns = {'created_at': 'Time Created', 'text': 'Tweet Text', 'source': 'Source'\n",
    "                    , 'possibly_sensitive': 'Possibly Sensitive', 'lang': 'Language'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv('Impeach_tweets.csv', index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([nan, False, True], dtype=object)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['Possibly Sensitive'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "cleaned_df = df.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Time Created</th>\n",
       "      <th>Tweet Text</th>\n",
       "      <th>Source</th>\n",
       "      <th>Possibly Sensitive</th>\n",
       "      <th>Language</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2021-02-09 17:01:28+00:00</td>\n",
       "      <td>RT @JesseBWatters: If you put Maxine Waters wo...</td>\n",
       "      <td>Twitter for iPhone</td>\n",
       "      <td>NaN</td>\n",
       "      <td>en</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2021-02-09 17:01:28+00:00</td>\n",
       "      <td>Lindsey Graham Threatens to Call Democrats as ...</td>\n",
       "      <td>Twitter Web App</td>\n",
       "      <td>False</td>\n",
       "      <td>en</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2021-02-09 17:01:28+00:00</td>\n",
       "      <td>@NunesAlt As long as it's not Trump streaking....</td>\n",
       "      <td>Twitter Web App</td>\n",
       "      <td>NaN</td>\n",
       "      <td>en</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2021-02-09 17:01:28+00:00</td>\n",
       "      <td>RT @chipfranklin: Who else thinks the vote to ...</td>\n",
       "      <td>Twitter Web App</td>\n",
       "      <td>NaN</td>\n",
       "      <td>en</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2021-02-09 17:01:28+00:00</td>\n",
       "      <td>RT @JaySekulow: HAPPENING NOW: Impeachment Tri...</td>\n",
       "      <td>Twitter for iPhone</td>\n",
       "      <td>False</td>\n",
       "      <td>en</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99996</th>\n",
       "      <td>2021-02-09 17:36:53+00:00</td>\n",
       "      <td>RT @FranklinH3000: It took Martin van Buren, h...</td>\n",
       "      <td>TweetDeck</td>\n",
       "      <td>NaN</td>\n",
       "      <td>en</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99997</th>\n",
       "      <td>2021-02-09 17:36:53+00:00</td>\n",
       "      <td>RT @jason_meister: This #impeachment trial is ...</td>\n",
       "      <td>Twitter for iPhone</td>\n",
       "      <td>NaN</td>\n",
       "      <td>en</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99998</th>\n",
       "      <td>2021-02-09 17:36:53+00:00</td>\n",
       "      <td>RT @chipfranklin: Who else thinks the vote to ...</td>\n",
       "      <td>Twitter for iPhone</td>\n",
       "      <td>NaN</td>\n",
       "      <td>en</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99999</th>\n",
       "      <td>2021-02-09 17:36:53+00:00</td>\n",
       "      <td>The first day of the Senate impeachment trial ...</td>\n",
       "      <td>SocialFlow</td>\n",
       "      <td>False</td>\n",
       "      <td>en</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100000</th>\n",
       "      <td>2021-02-09 17:36:53+00:00</td>\n",
       "      <td>https://t.co/usj6xbBti0</td>\n",
       "      <td>Twitter Web App</td>\n",
       "      <td>False</td>\n",
       "      <td>und</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>100001 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                    Time Created  \\\n",
       "0      2021-02-09 17:01:28+00:00   \n",
       "1      2021-02-09 17:01:28+00:00   \n",
       "2      2021-02-09 17:01:28+00:00   \n",
       "3      2021-02-09 17:01:28+00:00   \n",
       "4      2021-02-09 17:01:28+00:00   \n",
       "...                          ...   \n",
       "99996  2021-02-09 17:36:53+00:00   \n",
       "99997  2021-02-09 17:36:53+00:00   \n",
       "99998  2021-02-09 17:36:53+00:00   \n",
       "99999  2021-02-09 17:36:53+00:00   \n",
       "100000 2021-02-09 17:36:53+00:00   \n",
       "\n",
       "                                               Tweet Text              Source  \\\n",
       "0       RT @JesseBWatters: If you put Maxine Waters wo...  Twitter for iPhone   \n",
       "1       Lindsey Graham Threatens to Call Democrats as ...     Twitter Web App   \n",
       "2       @NunesAlt As long as it's not Trump streaking....     Twitter Web App   \n",
       "3       RT @chipfranklin: Who else thinks the vote to ...     Twitter Web App   \n",
       "4       RT @JaySekulow: HAPPENING NOW: Impeachment Tri...  Twitter for iPhone   \n",
       "...                                                   ...                 ...   \n",
       "99996   RT @FranklinH3000: It took Martin van Buren, h...           TweetDeck   \n",
       "99997   RT @jason_meister: This #impeachment trial is ...  Twitter for iPhone   \n",
       "99998   RT @chipfranklin: Who else thinks the vote to ...  Twitter for iPhone   \n",
       "99999   The first day of the Senate impeachment trial ...          SocialFlow   \n",
       "100000                            https://t.co/usj6xbBti0     Twitter Web App   \n",
       "\n",
       "       Possibly Sensitive Language  \n",
       "0                     NaN       en  \n",
       "1                   False       en  \n",
       "2                     NaN       en  \n",
       "3                     NaN       en  \n",
       "4                   False       en  \n",
       "...                   ...      ...  \n",
       "99996                 NaN       en  \n",
       "99997                 NaN       en  \n",
       "99998                 NaN       en  \n",
       "99999               False       en  \n",
       "100000              False      und  \n",
       "\n",
       "[100001 rows x 5 columns]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cleaned_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-18-792820305062>:2: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  cleaned_df['Tweet Text'][i] = \" \".join([word for word in cleaned_df['Tweet Text'][i].split()\n"
     ]
    }
   ],
   "source": [
    "for i in range(len(cleaned_df['Tweet Text'])):\n",
    "    cleaned_df['Tweet Text'][i] = \" \".join([word for word in cleaned_df['Tweet Text'][i].split()\n",
    "                              if 'http' not in word and '@' not in word and '<' not in word])\n",
    "\n",
    "# remove special characters\n",
    "cleaned_df['Tweet Text'] = cleaned_df['Tweet Text'].apply(lambda x: re.sub('[!@#$:).;,?&-]', '', x))\n",
    "cleaned_df['Tweet Text'] = cleaned_df['Tweet Text'].apply(lambda x: re.sub('  ', ' ', x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "cleaned_df = cleaned_df[cleaned_df['Possibly Sensitive'] != True]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Time Created</th>\n",
       "      <th>Tweet Text</th>\n",
       "      <th>Source</th>\n",
       "      <th>Possibly Sensitive</th>\n",
       "      <th>Language</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2021-02-09 17:01:28+00:00</td>\n",
       "      <td>RT If you put Maxine Waters words in Trump's m...</td>\n",
       "      <td>Twitter for iPhone</td>\n",
       "      <td>NaN</td>\n",
       "      <td>en</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2021-02-09 17:01:28+00:00</td>\n",
       "      <td>Lindsey Graham Threatens to Call Democrats as ...</td>\n",
       "      <td>Twitter Web App</td>\n",
       "      <td>False</td>\n",
       "      <td>en</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2021-02-09 17:01:28+00:00</td>\n",
       "      <td>As long as it's not Trump streaking I don't th...</td>\n",
       "      <td>Twitter Web App</td>\n",
       "      <td>NaN</td>\n",
       "      <td>en</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2021-02-09 17:01:28+00:00</td>\n",
       "      <td>RT Who else thinks the vote to convict Trump i...</td>\n",
       "      <td>Twitter Web App</td>\n",
       "      <td>NaN</td>\n",
       "      <td>en</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2021-02-09 17:01:28+00:00</td>\n",
       "      <td>RT HAPPENING NOW Impeachment Trial Begins</td>\n",
       "      <td>Twitter for iPhone</td>\n",
       "      <td>False</td>\n",
       "      <td>en</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99996</th>\n",
       "      <td>2021-02-09 17:36:53+00:00</td>\n",
       "      <td>RT It took Martin van Buren himself not a stri...</td>\n",
       "      <td>TweetDeck</td>\n",
       "      <td>NaN</td>\n",
       "      <td>en</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99997</th>\n",
       "      <td>2021-02-09 17:36:53+00:00</td>\n",
       "      <td>RT This impeachment trial is proof that we liv...</td>\n",
       "      <td>Twitter for iPhone</td>\n",
       "      <td>NaN</td>\n",
       "      <td>en</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99998</th>\n",
       "      <td>2021-02-09 17:36:53+00:00</td>\n",
       "      <td>RT Who else thinks the vote to convict Trump i...</td>\n",
       "      <td>Twitter for iPhone</td>\n",
       "      <td>NaN</td>\n",
       "      <td>en</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99999</th>\n",
       "      <td>2021-02-09 17:36:53+00:00</td>\n",
       "      <td>The first day of the Senate impeachment trial ...</td>\n",
       "      <td>SocialFlow</td>\n",
       "      <td>False</td>\n",
       "      <td>en</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100000</th>\n",
       "      <td>2021-02-09 17:36:53+00:00</td>\n",
       "      <td></td>\n",
       "      <td>Twitter Web App</td>\n",
       "      <td>False</td>\n",
       "      <td>und</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>99463 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                    Time Created  \\\n",
       "0      2021-02-09 17:01:28+00:00   \n",
       "1      2021-02-09 17:01:28+00:00   \n",
       "2      2021-02-09 17:01:28+00:00   \n",
       "3      2021-02-09 17:01:28+00:00   \n",
       "4      2021-02-09 17:01:28+00:00   \n",
       "...                          ...   \n",
       "99996  2021-02-09 17:36:53+00:00   \n",
       "99997  2021-02-09 17:36:53+00:00   \n",
       "99998  2021-02-09 17:36:53+00:00   \n",
       "99999  2021-02-09 17:36:53+00:00   \n",
       "100000 2021-02-09 17:36:53+00:00   \n",
       "\n",
       "                                               Tweet Text              Source  \\\n",
       "0       RT If you put Maxine Waters words in Trump's m...  Twitter for iPhone   \n",
       "1       Lindsey Graham Threatens to Call Democrats as ...     Twitter Web App   \n",
       "2       As long as it's not Trump streaking I don't th...     Twitter Web App   \n",
       "3       RT Who else thinks the vote to convict Trump i...     Twitter Web App   \n",
       "4               RT HAPPENING NOW Impeachment Trial Begins  Twitter for iPhone   \n",
       "...                                                   ...                 ...   \n",
       "99996   RT It took Martin van Buren himself not a stri...           TweetDeck   \n",
       "99997   RT This impeachment trial is proof that we liv...  Twitter for iPhone   \n",
       "99998   RT Who else thinks the vote to convict Trump i...  Twitter for iPhone   \n",
       "99999   The first day of the Senate impeachment trial ...          SocialFlow   \n",
       "100000                                                        Twitter Web App   \n",
       "\n",
       "       Possibly Sensitive Language  \n",
       "0                     NaN       en  \n",
       "1                   False       en  \n",
       "2                     NaN       en  \n",
       "3                     NaN       en  \n",
       "4                   False       en  \n",
       "...                   ...      ...  \n",
       "99996                 NaN       en  \n",
       "99997                 NaN       en  \n",
       "99998                 NaN       en  \n",
       "99999               False       en  \n",
       "100000              False      und  \n",
       "\n",
       "[99463 rows x 5 columns]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cleaned_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "cleaned_df.to_csv('Impeach_tweets_clean.csv', index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "sentiment_df = cleaned_df.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "sentiment_df[['polarity', 'subjectivity']] = sentiment_df['Tweet Text'].apply(lambda text: pd.Series(TextBlob(text).sentiment))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "conditions = [(sentiment_df['polarity'] > 0),\n",
    "              (sentiment_df['polarity'] == 0),\n",
    "              (sentiment_df['polarity'] < 0)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "values = ['positive', 'neutral', 'negative']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "sentiment_df['sentiment'] = np.select(conditions, values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Time Created</th>\n",
       "      <th>Tweet Text</th>\n",
       "      <th>Source</th>\n",
       "      <th>Possibly Sensitive</th>\n",
       "      <th>Language</th>\n",
       "      <th>polarity</th>\n",
       "      <th>subjectivity</th>\n",
       "      <th>sentiment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2021-02-09 17:01:28+00:00</td>\n",
       "      <td>RT If you put Maxine Waters words in Trump's m...</td>\n",
       "      <td>Twitter for iPhone</td>\n",
       "      <td>NaN</td>\n",
       "      <td>en</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2021-02-09 17:01:28+00:00</td>\n",
       "      <td>Lindsey Graham Threatens to Call Democrats as ...</td>\n",
       "      <td>Twitter Web App</td>\n",
       "      <td>False</td>\n",
       "      <td>en</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2021-02-09 17:01:28+00:00</td>\n",
       "      <td>As long as it's not Trump streaking I don't th...</td>\n",
       "      <td>Twitter Web App</td>\n",
       "      <td>NaN</td>\n",
       "      <td>en</td>\n",
       "      <td>-0.05</td>\n",
       "      <td>0.4</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2021-02-09 17:01:28+00:00</td>\n",
       "      <td>RT Who else thinks the vote to convict Trump i...</td>\n",
       "      <td>Twitter Web App</td>\n",
       "      <td>NaN</td>\n",
       "      <td>en</td>\n",
       "      <td>-0.40</td>\n",
       "      <td>0.7</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2021-02-09 17:01:28+00:00</td>\n",
       "      <td>RT HAPPENING NOW Impeachment Trial Begins</td>\n",
       "      <td>Twitter for iPhone</td>\n",
       "      <td>False</td>\n",
       "      <td>en</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>neutral</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               Time Created  \\\n",
       "0 2021-02-09 17:01:28+00:00   \n",
       "1 2021-02-09 17:01:28+00:00   \n",
       "2 2021-02-09 17:01:28+00:00   \n",
       "3 2021-02-09 17:01:28+00:00   \n",
       "4 2021-02-09 17:01:28+00:00   \n",
       "\n",
       "                                          Tweet Text              Source  \\\n",
       "0  RT If you put Maxine Waters words in Trump's m...  Twitter for iPhone   \n",
       "1  Lindsey Graham Threatens to Call Democrats as ...     Twitter Web App   \n",
       "2  As long as it's not Trump streaking I don't th...     Twitter Web App   \n",
       "3  RT Who else thinks the vote to convict Trump i...     Twitter Web App   \n",
       "4          RT HAPPENING NOW Impeachment Trial Begins  Twitter for iPhone   \n",
       "\n",
       "  Possibly Sensitive Language  polarity  subjectivity sentiment  \n",
       "0                NaN       en      0.00           0.0   neutral  \n",
       "1              False       en      0.00           0.0   neutral  \n",
       "2                NaN       en     -0.05           0.4  negative  \n",
       "3                NaN       en     -0.40           0.7  negative  \n",
       "4              False       en      0.00           0.0   neutral  "
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentiment_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "sentiment_df.to_csv('Impeach_tweets_sentiment.csv', index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}