fakenewsplayground-Lavin.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>author</th>\n",
       "      <th>title</th>\n",
       "      <th>text</th>\n",
       "      <th>site_url</th>\n",
       "      <th>country</th>\n",
       "      <th>domain_rank</th>\n",
       "      <th>thread_title</th>\n",
       "      <th>spam_score</th>\n",
       "      <th>main_img_url</th>\n",
       "      <th>replies_count</th>\n",
       "      <th>participants_count</th>\n",
       "      <th>likes</th>\n",
       "      <th>comments</th>\n",
       "      <th>shares</th>\n",
       "      <th>type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Barracuda Brigade</td>\n",
       "      <td>Muslims BUSTED: They Stole Millions In Gov’t B...</td>\n",
       "      <td>Print They should pay all the back all the mon...</td>\n",
       "      <td>100percentfedup.com</td>\n",
       "      <td>US</td>\n",
       "      <td>25689.0</td>\n",
       "      <td>Muslims BUSTED: They Stole Millions In Gov’t B...</td>\n",
       "      <td>0.000</td>\n",
       "      <td>http://bb4sp.com/wp-content/uploads/2016/10/Fu...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>bias</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>reasoning with facts</td>\n",
       "      <td>Re: Why Did Attorney General Loretta Lynch Ple...</td>\n",
       "      <td>Why Did Attorney General Loretta Lynch Plead T...</td>\n",
       "      <td>100percentfedup.com</td>\n",
       "      <td>US</td>\n",
       "      <td>25689.0</td>\n",
       "      <td>Re: Why Did Attorney General Loretta Lynch Ple...</td>\n",
       "      <td>0.000</td>\n",
       "      <td>http://bb4sp.com/wp-content/uploads/2016/10/Fu...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>bias</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Barracuda Brigade</td>\n",
       "      <td>BREAKING: Weiner Cooperating With FBI On Hilla...</td>\n",
       "      <td>Red State : \\nFox News Sunday reported this mo...</td>\n",
       "      <td>100percentfedup.com</td>\n",
       "      <td>US</td>\n",
       "      <td>25689.0</td>\n",
       "      <td>BREAKING: Weiner Cooperating With FBI On Hilla...</td>\n",
       "      <td>0.000</td>\n",
       "      <td>http://bb4sp.com/wp-content/uploads/2016/10/Fu...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>bias</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Fed Up</td>\n",
       "      <td>PIN DROP SPEECH BY FATHER OF DAUGHTER Kidnappe...</td>\n",
       "      <td>Email Kayla Mueller was a prisoner and torture...</td>\n",
       "      <td>100percentfedup.com</td>\n",
       "      <td>US</td>\n",
       "      <td>25689.0</td>\n",
       "      <td>PIN DROP SPEECH BY FATHER OF DAUGHTER Kidnappe...</td>\n",
       "      <td>0.068</td>\n",
       "      <td>http://100percentfedup.com/wp-content/uploads/...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>bias</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Fed Up</td>\n",
       "      <td>FANTASTIC! TRUMP'S 7 POINT PLAN To Reform Heal...</td>\n",
       "      <td>Email HEALTHCARE REFORM TO MAKE AMERICA GREAT ...</td>\n",
       "      <td>100percentfedup.com</td>\n",
       "      <td>US</td>\n",
       "      <td>25689.0</td>\n",
       "      <td>FANTASTIC! TRUMP'S 7 POINT PLAN To Reform Heal...</td>\n",
       "      <td>0.865</td>\n",
       "      <td>http://100percentfedup.com/wp-content/uploads/...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>bias</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Barracuda Brigade</td>\n",
       "      <td>Hillary Goes Absolutely Berserk On Protester A...</td>\n",
       "      <td>Print Hillary goes absolutely berserk! She exp...</td>\n",
       "      <td>100percentfedup.com</td>\n",
       "      <td>US</td>\n",
       "      <td>25689.0</td>\n",
       "      <td>Hillary Goes Absolutely Berserk On Protester A...</td>\n",
       "      <td>0.000</td>\n",
       "      <td>http://bb4sp.com/wp-content/uploads/2016/11/Fu...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>bias</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Fed Up</td>\n",
       "      <td>BREAKING! NYPD Ready To Make Arrests In Weiner...</td>\n",
       "      <td>BREAKING! NYPD Ready To Make Arrests In Weiner...</td>\n",
       "      <td>100percentfedup.com</td>\n",
       "      <td>US</td>\n",
       "      <td>25689.0</td>\n",
       "      <td>BREAKING! NYPD Ready To Make Arrests In Weiner...</td>\n",
       "      <td>0.701</td>\n",
       "      <td>http://100percentfedup.com/wp-content/uploads/...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>bias</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Fed Up</td>\n",
       "      <td>WOW! WHISTLEBLOWER TELLS CHILLING STORY Of Mas...</td>\n",
       "      <td>BREAKING! NYPD Ready To Make Arrests In Weiner...</td>\n",
       "      <td>100percentfedup.com</td>\n",
       "      <td>US</td>\n",
       "      <td>25689.0</td>\n",
       "      <td>WOW! WHISTLEBLOWER TELLS CHILLING STORY Of Mas...</td>\n",
       "      <td>0.188</td>\n",
       "      <td>http://100percentfedup.com/wp-content/uploads/...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>bias</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Fed Up</td>\n",
       "      <td>BREAKING: CLINTON CLEARED...Was This A Coordin...</td>\n",
       "      <td>\\nLimbaugh said that the revelations in the Wi...</td>\n",
       "      <td>100percentfedup.com</td>\n",
       "      <td>US</td>\n",
       "      <td>25689.0</td>\n",
       "      <td>BREAKING: CLINTON CLEARED...Was This A Coordin...</td>\n",
       "      <td>0.144</td>\n",
       "      <td>http://100percentfedup.com/wp-content/uploads/...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>bias</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Fed Up</td>\n",
       "      <td>EVIL HILLARY SUPPORTERS Yell \"F*ck Trump\"…Burn...</td>\n",
       "      <td>Email \\nThese people are sick and evil. They w...</td>\n",
       "      <td>100percentfedup.com</td>\n",
       "      <td>US</td>\n",
       "      <td>25689.0</td>\n",
       "      <td>EVIL HILLARY SUPPORTERS Yell \"F*ck Trump\"…Burn...</td>\n",
       "      <td>0.995</td>\n",
       "      <td>http://100percentfedup.com/wp-content/uploads/...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>bias</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 author                                              title  \\\n",
       "0     Barracuda Brigade  Muslims BUSTED: They Stole Millions In Gov’t B...   \n",
       "1  reasoning with facts  Re: Why Did Attorney General Loretta Lynch Ple...   \n",
       "2     Barracuda Brigade  BREAKING: Weiner Cooperating With FBI On Hilla...   \n",
       "3                Fed Up  PIN DROP SPEECH BY FATHER OF DAUGHTER Kidnappe...   \n",
       "4                Fed Up  FANTASTIC! TRUMP'S 7 POINT PLAN To Reform Heal...   \n",
       "5     Barracuda Brigade  Hillary Goes Absolutely Berserk On Protester A...   \n",
       "6                Fed Up  BREAKING! NYPD Ready To Make Arrests In Weiner...   \n",
       "7                Fed Up  WOW! WHISTLEBLOWER TELLS CHILLING STORY Of Mas...   \n",
       "8                Fed Up  BREAKING: CLINTON CLEARED...Was This A Coordin...   \n",
       "9                Fed Up  EVIL HILLARY SUPPORTERS Yell \"F*ck Trump\"…Burn...   \n",
       "\n",
       "                                                text             site_url  \\\n",
       "0  Print They should pay all the back all the mon...  100percentfedup.com   \n",
       "1  Why Did Attorney General Loretta Lynch Plead T...  100percentfedup.com   \n",
       "2  Red State : \\nFox News Sunday reported this mo...  100percentfedup.com   \n",
       "3  Email Kayla Mueller was a prisoner and torture...  100percentfedup.com   \n",
       "4  Email HEALTHCARE REFORM TO MAKE AMERICA GREAT ...  100percentfedup.com   \n",
       "5  Print Hillary goes absolutely berserk! She exp...  100percentfedup.com   \n",
       "6  BREAKING! NYPD Ready To Make Arrests In Weiner...  100percentfedup.com   \n",
       "7  BREAKING! NYPD Ready To Make Arrests In Weiner...  100percentfedup.com   \n",
       "8  \\nLimbaugh said that the revelations in the Wi...  100percentfedup.com   \n",
       "9  Email \\nThese people are sick and evil. They w...  100percentfedup.com   \n",
       "\n",
       "  country  domain_rank                                       thread_title  \\\n",
       "0      US      25689.0  Muslims BUSTED: They Stole Millions In Gov’t B...   \n",
       "1      US      25689.0  Re: Why Did Attorney General Loretta Lynch Ple...   \n",
       "2      US      25689.0  BREAKING: Weiner Cooperating With FBI On Hilla...   \n",
       "3      US      25689.0  PIN DROP SPEECH BY FATHER OF DAUGHTER Kidnappe...   \n",
       "4      US      25689.0  FANTASTIC! TRUMP'S 7 POINT PLAN To Reform Heal...   \n",
       "5      US      25689.0  Hillary Goes Absolutely Berserk On Protester A...   \n",
       "6      US      25689.0  BREAKING! NYPD Ready To Make Arrests In Weiner...   \n",
       "7      US      25689.0  WOW! WHISTLEBLOWER TELLS CHILLING STORY Of Mas...   \n",
       "8      US      25689.0  BREAKING: CLINTON CLEARED...Was This A Coordin...   \n",
       "9      US      25689.0  EVIL HILLARY SUPPORTERS Yell \"F*ck Trump\"…Burn...   \n",
       "\n",
       "   spam_score                                       main_img_url  \\\n",
       "0       0.000  http://bb4sp.com/wp-content/uploads/2016/10/Fu...   \n",
       "1       0.000  http://bb4sp.com/wp-content/uploads/2016/10/Fu...   \n",
       "2       0.000  http://bb4sp.com/wp-content/uploads/2016/10/Fu...   \n",
       "3       0.068  http://100percentfedup.com/wp-content/uploads/...   \n",
       "4       0.865  http://100percentfedup.com/wp-content/uploads/...   \n",
       "5       0.000  http://bb4sp.com/wp-content/uploads/2016/11/Fu...   \n",
       "6       0.701  http://100percentfedup.com/wp-content/uploads/...   \n",
       "7       0.188  http://100percentfedup.com/wp-content/uploads/...   \n",
       "8       0.144  http://100percentfedup.com/wp-content/uploads/...   \n",
       "9       0.995  http://100percentfedup.com/wp-content/uploads/...   \n",
       "\n",
       "   replies_count  participants_count  likes  comments  shares  type  \n",
       "0              0                   1      0         0       0  bias  \n",
       "1              0                   1      0         0       0  bias  \n",
       "2              0                   1      0         0       0  bias  \n",
       "3              0                   0      0         0       0  bias  \n",
       "4              0                   0      0         0       0  bias  \n",
       "5              0                   1      0         0       0  bias  \n",
       "6              0                   0      0         0       0  bias  \n",
       "7              0                   0      0         0       0  bias  \n",
       "8              0                   0      0         0       0  bias  \n",
       "9              0                   0      0         0       0  bias  "
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "#from pandas import DataFrame, Series\n",
    "#import matplotlib.pyplot as plt\n",
    "\n",
    "frame = pd.read_csv(\"../fake.csv\")\n",
    "frame.drop(['uuid', 'ord_in_thread','published','language','crawled'], axis=1, inplace=True)\n",
    "frame.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Types and counts of stories\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "type\n",
       "bias            443\n",
       "bs            11492\n",
       "conspiracy      430\n",
       "fake             19\n",
       "hate            246\n",
       "junksci         102\n",
       "satire          146\n",
       "state           121\n",
       "dtype: int64"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(\"Types and counts of stories\")\n",
    "df = frame.groupby([\"type\"]).size()\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Print They should pay all the back all the mon...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Why Did Attorney General Loretta Lynch Plead T...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Red State : \\nFox News Sunday reported this mo...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Email Kayla Mueller was a prisoner and torture...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Email HEALTHCARE REFORM TO MAKE AMERICA GREAT ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Print Hillary goes absolutely berserk! She exp...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>BREAKING! NYPD Ready To Make Arrests In Weiner...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>BREAKING! NYPD Ready To Make Arrests In Weiner...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>\\nLimbaugh said that the revelations in the Wi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Email \\nThese people are sick and evil. They w...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>\\nWho? Comedian. \\nWhere would she move? Spain...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Students expressed their “fear” over a Trump p...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>Email For Republican politicians like Ohio Gov...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Copyright © 2016 100PercentFedUp.com, in assoc...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>Go to Article A Trump supporter wearing a Trum...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>Copyright © 2016 100PercentFedUp.com, in assoc...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>Go to Article Donald Trump was willing to give...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>John McNaughton is a special American painter ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>Go to Article Dear Abby, I supported a woman I...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>Mark Cuban has made no secret of his dislike f...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>David Wilcox, a 49 year old Chicago man who wa...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>Go to Article Filmmaker Ami Horowitz takes to ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>Go to Article \\nPolitical activist and Hillary...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>Go to Article Rafael Salamanca’s interview on ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>Go to Article In The Bubble, life continues fo...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>Go to Article \\nThey had to know they had it c...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>DONALD TRUMP SKIPS MEDIA: Delivers Transition ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>Go to Article The left is shameful in their la...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>BOOM! Kellyanne Conway Shuts Down CNN’s Cuomo ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12969</th>\n",
       "      <td>Stock \"On Fire!\"</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12970</th>\n",
       "      <td>more like a burning bag of shit on a doorstep....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12971</th>\n",
       "      <td>Everyone loved the SpaceX video of spaceship g...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12972</th>\n",
       "      <td>non gaap has always been bullshit.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12973</th>\n",
       "      <td>Generally Confusing Accounting Principles</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12974</th>\n",
       "      <td>All this non-GAAP activity should be disclosed...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12975</th>\n",
       "      <td>Also, since when did we start using \"non-GAAP\"...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12976</th>\n",
       "      <td>.. \"SEC in  recent months has raised concern t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12977</th>\n",
       "      <td>I'm sure they drastically changed accounting m...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12978</th>\n",
       "      <td>Tesla's on FIRE! \\noh wait</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12979</th>\n",
       "      <td>Musk's cash furnace is the posterchild for IB ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12980</th>\n",
       "      <td>Non-GAAP = SEC* FRAUD\\nNon-GAAP = SEC* SWINDLE...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12981</th>\n",
       "      <td>Can we shoot this delusional fuck to Mars! ?</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12982</th>\n",
       "      <td>A dollar to be made and all I got was a quarte...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12983</th>\n",
       "      <td>when i awoke this morning the Dow Jones Propag...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12984</th>\n",
       "      <td>Then you beat me by a quarter because all I go...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12985</th>\n",
       "      <td>And for the last two weeks, Gold is \"naturally...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12986</th>\n",
       "      <td>I love that video clip. I think it is a male v...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12987</th>\n",
       "      <td>A big selloff into the election = TRUMP 2016! ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12988</th>\n",
       "      <td>Technical explanations are out of favor, so he...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12989</th>\n",
       "      <td>Nope its oil.  Thank you for my technical win ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12990</th>\n",
       "      <td>The weekly Wednesday Oil follies from 10:20 (s...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12991</th>\n",
       "      <td>Chipotle.coli Mexican Grill weighing on restau...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12992</th>\n",
       "      <td>Ah the old pump and dump.  The market should b...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12993</th>\n",
       "      <td>A messaging app is worth $40 billion dollars i...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12994</th>\n",
       "      <td>It DOES allow you to put a dog face on top of ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12995</th>\n",
       "      <td>Wait till you see what happens to the valuatio...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12996</th>\n",
       "      <td>I'm waiting for the one that puts a pussy on m...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12997</th>\n",
       "      <td>$4 Billion even after they are known to be kee...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12998</th>\n",
       "      <td>of course - how else would they disceminate te...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>12999 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                    text\n",
       "0      Print They should pay all the back all the mon...\n",
       "1      Why Did Attorney General Loretta Lynch Plead T...\n",
       "2      Red State : \\nFox News Sunday reported this mo...\n",
       "3      Email Kayla Mueller was a prisoner and torture...\n",
       "4      Email HEALTHCARE REFORM TO MAKE AMERICA GREAT ...\n",
       "5      Print Hillary goes absolutely berserk! She exp...\n",
       "6      BREAKING! NYPD Ready To Make Arrests In Weiner...\n",
       "7      BREAKING! NYPD Ready To Make Arrests In Weiner...\n",
       "8      \\nLimbaugh said that the revelations in the Wi...\n",
       "9      Email \\nThese people are sick and evil. They w...\n",
       "10                                                      \n",
       "11     \\nWho? Comedian. \\nWhere would she move? Spain...\n",
       "12     Students expressed their “fear” over a Trump p...\n",
       "13     Email For Republican politicians like Ohio Gov...\n",
       "14     Copyright © 2016 100PercentFedUp.com, in assoc...\n",
       "15     Go to Article A Trump supporter wearing a Trum...\n",
       "16     Copyright © 2016 100PercentFedUp.com, in assoc...\n",
       "17     Go to Article Donald Trump was willing to give...\n",
       "18     John McNaughton is a special American painter ...\n",
       "19     Go to Article Dear Abby, I supported a woman I...\n",
       "20     Mark Cuban has made no secret of his dislike f...\n",
       "21     David Wilcox, a 49 year old Chicago man who wa...\n",
       "22     Go to Article Filmmaker Ami Horowitz takes to ...\n",
       "23     Go to Article \\nPolitical activist and Hillary...\n",
       "24     Go to Article Rafael Salamanca’s interview on ...\n",
       "25     Go to Article In The Bubble, life continues fo...\n",
       "26     Go to Article \\nThey had to know they had it c...\n",
       "27     DONALD TRUMP SKIPS MEDIA: Delivers Transition ...\n",
       "28     Go to Article The left is shameful in their la...\n",
       "29     BOOM! Kellyanne Conway Shuts Down CNN’s Cuomo ...\n",
       "...                                                  ...\n",
       "12969                                   Stock \"On Fire!\"\n",
       "12970  more like a burning bag of shit on a doorstep....\n",
       "12971  Everyone loved the SpaceX video of spaceship g...\n",
       "12972                 non gaap has always been bullshit.\n",
       "12973          Generally Confusing Accounting Principles\n",
       "12974  All this non-GAAP activity should be disclosed...\n",
       "12975  Also, since when did we start using \"non-GAAP\"...\n",
       "12976  .. \"SEC in  recent months has raised concern t...\n",
       "12977  I'm sure they drastically changed accounting m...\n",
       "12978                         Tesla's on FIRE! \\noh wait\n",
       "12979  Musk's cash furnace is the posterchild for IB ...\n",
       "12980  Non-GAAP = SEC* FRAUD\\nNon-GAAP = SEC* SWINDLE...\n",
       "12981       Can we shoot this delusional fuck to Mars! ?\n",
       "12982  A dollar to be made and all I got was a quarte...\n",
       "12983  when i awoke this morning the Dow Jones Propag...\n",
       "12984  Then you beat me by a quarter because all I go...\n",
       "12985  And for the last two weeks, Gold is \"naturally...\n",
       "12986  I love that video clip. I think it is a male v...\n",
       "12987  A big selloff into the election = TRUMP 2016! ...\n",
       "12988  Technical explanations are out of favor, so he...\n",
       "12989  Nope its oil.  Thank you for my technical win ...\n",
       "12990  The weekly Wednesday Oil follies from 10:20 (s...\n",
       "12991  Chipotle.coli Mexican Grill weighing on restau...\n",
       "12992  Ah the old pump and dump.  The market should b...\n",
       "12993  A messaging app is worth $40 billion dollars i...\n",
       "12994  It DOES allow you to put a dog face on top of ...\n",
       "12995  Wait till you see what happens to the valuatio...\n",
       "12996  I'm waiting for the one that puts a pussy on m...\n",
       "12997  $4 Billion even after they are known to be kee...\n",
       "12998  of course - how else would they disceminate te...\n",
       "\n",
       "[12999 rows x 1 columns]"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "#from pandas import DataFrame, Series\n",
    "#import matplotlib.pyplot as plt\n",
    "\n",
    "frame = pd.read_csv(\"../fake.csv\")\n",
    "frame.drop(['uuid', 'ord_in_thread','published','language','crawled'], axis=1, inplace=True)\n",
    "df1 = frame.fillna('No data in title field')\n",
    "df1 = df1.filter(items=['text'])\n",
    "df1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#this variable is a list of titles, but some of the titles are strings and some are \"nan\" values from the dataframe. \n",
    "#filter out of the nan values to get the code below to work\n",
    "titles = df1['text'].tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import requests \n",
    "import enchant\n",
    "\n",
    "words = requests.get('http://ir.dcs.gla.ac.uk/resources/linguistic_utils/stop_words')\n",
    "stoplist1 = words.text.split(\"\\r\\n\")\n",
    "\n",
    "from nltk.corpus import stopwords\n",
    "stoplist2 = set(stopwords.words('english'))\n",
    "\n",
    "stoplist1.extend(stoplist2)\n",
    "\n",
    "fullstops = list(set(stoplist1))\n",
    "\n",
    "def remove_stops(stoplist, wordlist):\n",
    "    result = []\n",
    "    for i in wordlist:\n",
    "        if i not in stoplist:\n",
    "                result.append(i)\n",
    "    return result\n",
    "\n",
    "def spellcheck(wordlist):\n",
    "    result = []\n",
    "    d = enchant.Dict(\"en_US\")\n",
    "    for i in wordlist:\n",
    "        if d.check(i) or d.check(i.capitalize()):\n",
    "            result.append(i)\n",
    "         \n",
    "    return result\n",
    "def clean_text(list_of_texts):\n",
    "    fully_cleaned =[]\n",
    "    #normalize ocr errors\n",
    "    for i in list_of_texts:\n",
    "        #lowercase all\n",
    "        ocr_lower = i.lower()\n",
    "        #tokenize, remove punctuation and numbers, remove tabs, newlines, etc.\n",
    "        ocr_cleaner = ocr_lower.replace(\"\\n\", \" \").replace(\"\\t\", \" \")\n",
    "        ocr_tokens = ocr_cleaner.split(\" \")\n",
    "        \n",
    "        no_numbers_or_punct = []\n",
    "        for token in ocr_tokens:\n",
    "            if token.isalpha():\n",
    "                no_numbers_or_punct.append(token)\n",
    "            else:\n",
    "                \n",
    "                new_token = \"\"\n",
    "                for letter in token:\n",
    "                    if letter.isalpha():\n",
    "                        new_token += letter\n",
    "                if new_token != \"\":\n",
    "                    no_numbers_or_punct.append(new_token)  \n",
    "        #almost_ready_before_spellcheck = remove_stops(fullstops, no_numbers_or_punct)\n",
    "        #almost_ready = spellcheck(almost_ready_before_spellcheck)\n",
    "        #from nltk.stem import WordNetLemmatizer\n",
    "        #lemmatization\n",
    "        #wordnet_lemmatizer = WordNetLemmatizer()\n",
    "        #lemmas = []\n",
    "        #for token in almost_ready:\n",
    "        #    lemma = wordnet_lemmatizer.lemmatize(token)\n",
    "        #    lemmas.append(lemma)\n",
    "        \n",
    "        #ready = [i for i in lemmas if len(i) > 2]\n",
    "        #fully_cleaned.append(ready)\n",
    "        fully_cleaned.append(no_numbers_or_punct)\n",
    "    return fully_cleaned\n",
    "ocr_cleaned = clean_text(titles)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "ocr_counters = [Counter(i) for i in ocr_cleaned]\n",
    "\n",
    "from sklearn.feature_extraction import DictVectorizer\n",
    "from sklearn.feature_extraction.text import TfidfTransformer\n",
    "v = DictVectorizer()\n",
    "\n",
    "X = v.fit_transform(ocr_counters)\n",
    "y = TfidfTransformer()\n",
    "Z = y.fit_transform(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.03430843,  0.        ,  0.        , ...,  0.        ,\n",
       "        0.        ,  0.        ])"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scaled_vsm = Z.toarray()\n",
    "len(scaled_vsm[0])\n",
    "scaled_vsm[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "176337"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(v.get_feature_names())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'a'"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "v.get_feature_names()[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "celltoolbar": "Raw Cell Format",
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}