Toxic comment classfiication.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "train_data = pd.read_csv('/home/ubuntu/Downloads/train.csv')\n",
    "test_data = pd.read_csv('/home/ubuntu/Downloads/test.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "all_comments_list = list(train_data['comment_text'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "all_words = [comment.split(' ') for comment in all_comments_list]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "all_words_flattened = []\n",
    "for list_as_words in all_words:\n",
    "    all_words_flattened.extend(list_as_words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "occurences_counter = Counter(all_words_flattened)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('the', 267335),\n",
       " ('to', 173307),\n",
       " ('', 135015),\n",
       " ('of', 131942),\n",
       " ('and', 124039),\n",
       " ('a', 121406),\n",
       " ('I', 99455),\n",
       " ('is', 98841),\n",
       " ('you', 94183),\n",
       " ('that', 84784),\n",
       " ('in', 77899),\n",
       " ('for', 57870),\n",
       " ('it', 55477),\n",
       " ('on', 50736),\n",
       " ('not', 50728),\n",
       " ('be', 48660),\n",
       " ('this', 42684),\n",
       " ('have', 41688),\n",
       " ('as', 40848),\n",
       " ('are', 40091),\n",
       " ('with', 34142),\n",
       " ('your', 33178),\n",
       " ('was', 31659),\n",
       " ('or', 29119),\n",
       " ('but', 25350),\n",
       " ('an', 25164),\n",
       " ('from', 23439),\n",
       " ('by', 23180),\n",
       " ('my', 22812),\n",
       " ('at', 21995),\n",
       " ('article', 21933),\n",
       " ('\"', 21335),\n",
       " ('about', 20939),\n",
       " ('do', 19016),\n",
       " ('The', 18729),\n",
       " ('if', 18537),\n",
       " ('can', 18334),\n",
       " ('has', 18107),\n",
       " ('will', 16736),\n",
       " ('would', 16711),\n",
       " ('page', 16339),\n",
       " ('like', 15577),\n",
       " ('me', 15342),\n",
       " ('what', 15335),\n",
       " ('been', 14891),\n",
       " ('so', 14881),\n",
       " ('all', 14087),\n",
       " ('just', 13900),\n",
       " ('any', 13885),\n",
       " ('they', 13824),\n",
       " ('which', 13687),\n",
       " ('one', 13577),\n",
       " ('should', 13463),\n",
       " ('-', 13408),\n",
       " ('no', 13235),\n",
       " ('more', 12802),\n",
       " ('If', 12695),\n",
       " ('other', 12449),\n",
       " ('there', 12219),\n",
       " (\"don't\", 12155),\n",
       " ('some', 12027),\n",
       " ('he', 11989),\n",
       " ('who', 11888),\n",
       " ('talk', 11354),\n",
       " ('his', 11267),\n",
       " ('think', 11092),\n",
       " ('You', 10706),\n",
       " ('we', 10651),\n",
       " ('am', 10072),\n",
       " ('because', 10045),\n",
       " ('see', 9882),\n",
       " ('Wikipedia', 9713),\n",
       " ('only', 9650),\n",
       " ('know', 9575),\n",
       " ('were', 9073),\n",
       " (\"I'm\", 8880),\n",
       " ('edit', 8758),\n",
       " ('also', 8667),\n",
       " ('It', 8603),\n",
       " ('out', 8603),\n",
       " ('use', 8504),\n",
       " ('people', 8472),\n",
       " ('may', 8456),\n",
       " ('their', 8347),\n",
       " ('\"\\n\\n', 8320),\n",
       " ('than', 8241),\n",
       " ('when', 8219),\n",
       " ('being', 8217),\n",
       " ('then', 8019),\n",
       " ('up', 7957),\n",
       " ('Please', 7624),\n",
       " ('i', 7620),\n",
       " ('how', 7575),\n",
       " ('This', 7348),\n",
       " ('make', 7345),\n",
       " ('did', 7298),\n",
       " ('had', 7282),\n",
       " ('get', 7143),\n",
       " ('why', 6971),\n",
       " ('even', 6921),\n",
       " (\"it's\", 6894),\n",
       " ('very', 6827),\n",
       " ('please', 6648),\n",
       " ('want', 6352),\n",
       " ('articles', 6334),\n",
       " ('its', 6254),\n",
       " ('could', 6249),\n",
       " ('does', 6176),\n",
       " ('such', 6079),\n",
       " ('here', 6071),\n",
       " ('time', 5965),\n",
       " ('them', 5956),\n",
       " ('need', 5813),\n",
       " ('good', 5689),\n",
       " ('these', 5524),\n",
       " ('it.', 5523),\n",
       " ('YOU', 5513),\n",
       " ('information', 5423),\n",
       " ('into', 5224),\n",
       " ('made', 5164),\n",
       " ('find', 5109),\n",
       " ('A', 5071),\n",
       " ('used', 5036),\n",
       " ('first', 4959),\n",
       " ('say', 4921),\n",
       " ('much', 4892),\n",
       " ('really', 4879),\n",
       " ('many', 4863),\n",
       " ('same', 4840),\n",
       " ('where', 4830),\n",
       " ('name', 4799),\n",
       " ('those', 4785),\n",
       " ('most', 4669),\n",
       " ('add', 4645),\n",
       " ('page.', 4623),\n",
       " ('way', 4583),\n",
       " ('now', 4505),\n",
       " ('article.', 4470),\n",
       " ('And', 4396),\n",
       " (\"I've\", 4363),\n",
       " ('sources', 4323),\n",
       " ('still', 4319),\n",
       " ('new', 4303),\n",
       " ('going', 4293),\n",
       " ('section', 4223),\n",
       " ('go', 4221),\n",
       " ('after', 4207),\n",
       " ('read', 4195),\n",
       " ('look', 4183),\n",
       " ('someone', 4182),\n",
       " ('over', 4135),\n",
       " ('In', 4126),\n",
       " ('before', 4094),\n",
       " ('source', 4070),\n",
       " ('deletion', 4066),\n",
       " ('own', 4057),\n",
       " ('two', 4036),\n",
       " ('without', 4035),\n",
       " ('edits', 4018),\n",
       " ('you.', 4003),\n",
       " ('since', 3940),\n",
       " ('pages', 3939),\n",
       " ('user', 3918),\n",
       " ('deleted', 3916),\n",
       " ('our', 3900),\n",
       " ('discussion', 3886),\n",
       " ('take', 3868),\n",
       " ('editing', 3857),\n",
       " ('might', 3848),\n",
       " ('help', 3782),\n",
       " ('Thank', 3765),\n",
       " ('work', 3741),\n",
       " ('under', 3729),\n",
       " ('back', 3693),\n",
       " ('something', 3668),\n",
       " ('point', 3624),\n",
       " (\"doesn't\", 3587),\n",
       " ('added', 3573),\n",
       " ('But', 3562),\n",
       " ('put', 3468),\n",
       " ('well', 3444),\n",
       " ('said', 3441),\n",
       " ('link', 3430),\n",
       " ('image', 3417),\n",
       " ('Wikipedia.', 3416),\n",
       " ('seems', 3414),\n",
       " ('never', 3408),\n",
       " (',', 3396),\n",
       " ('ask', 3382),\n",
       " ('blocked', 3364),\n",
       " ('him', 3354),\n",
       " ('believe', 3347),\n",
       " ('fact', 3341),\n",
       " ('\\n\\nI', 3327),\n",
       " ('using', 3319),\n",
       " ('article,', 3309),\n",
       " ('sure', 3294),\n",
       " ('her', 3258),\n",
       " ('There', 3238),\n",
       " ('another', 3229),\n",
       " ('personal', 3221),\n",
       " ('page,', 3196),\n",
       " ('As', 3134),\n",
       " ('right', 3119),\n",
       " (\"you're\", 3093),\n",
       " ('against', 3065),\n",
       " ('.', 3042),\n",
       " ('•', 3039),\n",
       " ('What', 2983),\n",
       " (\"didn't\", 2959),\n",
       " ('both', 2953),\n",
       " ('place', 2951),\n",
       " ('editors', 2943),\n",
       " ('reason', 2913),\n",
       " (\"can't\", 2887),\n",
       " ('hope', 2886),\n",
       " ('better', 2886),\n",
       " ('actually', 2877),\n",
       " ('Your', 2877),\n",
       " ('done', 2870),\n",
       " ('too', 2867),\n",
       " ('feel', 2864),\n",
       " ('list', 2859),\n",
       " ('content', 2841),\n",
       " ('part', 2839),\n",
       " ('remove', 2831),\n",
       " (\"It's\", 2829),\n",
       " ('THE', 2825),\n",
       " ('it,', 2823),\n",
       " (\"I'll\", 2818),\n",
       " ('That', 2817),\n",
       " ('trying', 2813),\n",
       " ('\\n\\n', 2779),\n",
       " ('He', 2763),\n",
       " ('few', 2762),\n",
       " ('little', 2753),\n",
       " ('So', 2749),\n",
       " ('already', 2749),\n",
       " ('free', 2721),\n",
       " ('must', 2718),\n",
       " ('copyright', 2715),\n",
       " ('comment', 2707),\n",
       " ('fuck', 2692),\n",
       " ('links', 2684),\n",
       " ('give', 2663),\n",
       " ('anything', 2657),\n",
       " ('note', 2635),\n",
       " ('removed', 2624),\n",
       " ('making', 2613),\n",
       " ('things', 2607),\n",
       " ('stop', 2605),\n",
       " ('keep', 2592),\n",
       " ('change', 2583),\n",
       " ('understand', 2578),\n",
       " ('rather', 2577),\n",
       " ('person', 2533),\n",
       " ('block', 2532),\n",
       " ('nothing', 2530),\n",
       " ('speedy', 2515),\n",
       " ('last', 2515),\n",
       " ('different', 2506),\n",
       " ('while', 2481),\n",
       " ('she', 2478),\n",
       " ('\\n', 2470),\n",
       " ('continue', 2466),\n",
       " ('off', 2443),\n",
       " ('comments', 2435),\n",
       " ('Thanks', 2406),\n",
       " ('me.', 2405),\n",
       " ('wikipedia', 2397),\n",
       " ('come', 2397),\n",
       " ('got', 2368),\n",
       " ('question', 2348),\n",
       " ('here.', 2341),\n",
       " ('try', 2341),\n",
       " ('found', 2340),\n",
       " ('history', 2330),\n",
       " ('We', 2327),\n",
       " ('agree', 2322),\n",
       " ('anyone', 2319),\n",
       " ('reliable', 2310),\n",
       " ('adding', 2298),\n",
       " ('leave', 2292),\n",
       " ('simply', 2291),\n",
       " ('probably', 2289),\n",
       " ('long', 2280),\n",
       " (\"isn't\", 2267),\n",
       " ('between', 2262),\n",
       " ('tag', 2222),\n",
       " ('every', 2221),\n",
       " (\"that's\", 2216),\n",
       " ('let', 2211),\n",
       " ('doing', 2205),\n",
       " ('original', 2204),\n",
       " ('NIGGER', 2202),\n",
       " ('subject', 2195),\n",
       " ('case', 2194),\n",
       " ('mean', 2194),\n",
       " ('thing', 2193),\n",
       " ('through', 2174),\n",
       " ('you,', 2161),\n",
       " ('welcome', 2131),\n",
       " ('seem', 2129),\n",
       " ('delete', 2113),\n",
       " ('least', 2103),\n",
       " (')', 2089),\n",
       " ('How', 2079),\n",
       " ('write', 2068),\n",
       " ('needs', 2062),\n",
       " ('called', 2057),\n",
       " ('check', 2044),\n",
       " ('reference', 2040),\n",
       " ('thought', 2039),\n",
       " ('word', 2029),\n",
       " ('says', 2020),\n",
       " ('problem', 2017),\n",
       " ('quite', 2008),\n",
       " ('Do', 2007),\n",
       " ('—', 2004),\n",
       " ('text', 1997),\n",
       " ('best', 1989),\n",
       " ('far', 1988),\n",
       " ('My', 1972),\n",
       " ('lot', 1972),\n",
       " ('For', 1969),\n",
       " ('again', 1968),\n",
       " ('around', 1962),\n",
       " ('having', 1955),\n",
       " ('editor', 1953),\n",
       " ('always', 1949),\n",
       " ('show', 1944),\n",
       " ('IP', 1938),\n",
       " ('created', 1923),\n",
       " ('They', 1920),\n",
       " ('fair', 1908),\n",
       " ('\\n\\n\"', 1903),\n",
       " ('great', 1900),\n",
       " ('clearly', 1892),\n",
       " ('issue', 1888),\n",
       " ('whether', 1884),\n",
       " ('others', 1882),\n",
       " ('makes', 1874),\n",
       " ('consider', 1874),\n",
       " ('Talk', 1862),\n",
       " ('years', 1855),\n",
       " ('until', 1847),\n",
       " ('However,', 1846),\n",
       " ('(talk)', 1839),\n",
       " ('given', 1830),\n",
       " ('considered', 1828),\n",
       " ('me,', 1827),\n",
       " ('cannot', 1821),\n",
       " ('mention', 1818),\n",
       " ('reverted', 1810),\n",
       " ('ever', 1808),\n",
       " ('tell', 1803),\n",
       " ('|', 1801),\n",
       " ('Why', 1799),\n",
       " ('several', 1798),\n",
       " ('important', 1793),\n",
       " (\"I'd\", 1792),\n",
       " ('saying', 1789),\n",
       " ('wiki', 1781),\n",
       " ('policy', 1778),\n",
       " ('enough', 1778),\n",
       " ('number', 1775),\n",
       " ('bit', 1764),\n",
       " ('questions', 1763),\n",
       " ('us', 1760),\n",
       " ('that.', 1753),\n",
       " ('references', 1752),\n",
       " ('though', 1748),\n",
       " ('post', 1747),\n",
       " ('criteria', 1741),\n",
       " ('real', 1735),\n",
       " ('this.', 1728),\n",
       " ('above', 1728),\n",
       " ('wrong', 1714),\n",
       " ('clear', 1711),\n",
       " ('English', 1711),\n",
       " ('support', 1701),\n",
       " ('written', 1701),\n",
       " ('claim', 1690),\n",
       " ('listed', 1680),\n",
       " ('either', 1678),\n",
       " ('images', 1678),\n",
       " ('bad', 1675),\n",
       " ('each', 1666),\n",
       " ('material', 1663),\n",
       " ('&', 1662),\n",
       " ('based', 1661),\n",
       " ('include', 1657),\n",
       " ('No', 1652),\n",
       " ('evidence', 1640),\n",
       " ('request', 1633),\n",
       " ('current', 1624),\n",
       " ('consensus', 1624),\n",
       " ('matter', 1619),\n",
       " ('term', 1617),\n",
       " ('revert', 1615),\n",
       " ('vandalism', 1612),\n",
       " ('else', 1599),\n",
       " ('old', 1593),\n",
       " ('suggest', 1592),\n",
       " ('whole', 1590),\n",
       " ('following', 1588),\n",
       " ('message', 1584),\n",
       " ('To', 1579),\n",
       " ('Wikipedia,', 1579),\n",
       " ('down', 1574),\n",
       " ('state', 1569),\n",
       " ('create', 1557),\n",
       " ('users', 1555),\n",
       " ('book', 1551),\n",
       " ('top', 1550),\n",
       " ('that,', 1550),\n",
       " ('Also,', 1548),\n",
       " ('further', 1547),\n",
       " ('main', 1546),\n",
       " ('seen', 1545),\n",
       " ('site', 1542),\n",
       " ('IS', 1539),\n",
       " ('Is', 1537),\n",
       " ('SUCK', 1509),\n",
       " ('(and', 1508),\n",
       " ('means', 1508),\n",
       " ('three', 1507),\n",
       " ('u', 1501),\n",
       " ('kind', 1500),\n",
       " ('account', 1495),\n",
       " ('MY', 1491),\n",
       " ('Just', 1489),\n",
       " ('this,', 1487),\n",
       " ('provide', 1486),\n",
       " ('view', 1486),\n",
       " ('New', 1473),\n",
       " ('less', 1473),\n",
       " ('mentioned', 1463),\n",
       " ('once', 1458),\n",
       " ('yourself', 1453),\n",
       " (\"Wikipedia's\", 1453),\n",
       " ('2', 1452),\n",
       " ('left', 1447),\n",
       " ('instead', 1447),\n",
       " ('AND', 1437),\n",
       " ('is,', 1422),\n",
       " ('yet', 1422),\n",
       " ('call', 1419),\n",
       " ('opinion', 1417),\n",
       " ('able', 1414),\n",
       " ('start', 1414),\n",
       " ('NOT', 1409),\n",
       " ('version', 1408),\n",
       " ('getting', 1405),\n",
       " ('notice', 1401),\n",
       " ('All', 1394),\n",
       " ('here,', 1389),\n",
       " ('changes', 1384),\n",
       " ('removing', 1383),\n",
       " ('including', 1377),\n",
       " ('idea', 1374),\n",
       " ('correct', 1373),\n",
       " ('time.', 1369),\n",
       " ('them.', 1365),\n",
       " ('explain', 1359),\n",
       " ('title', 1356),\n",
       " ('rules', 1352),\n",
       " ('known', 1352),\n",
       " ('regarding', 1349),\n",
       " ('sign', 1349),\n",
       " ('ARE', 1340),\n",
       " ('language', 1333),\n",
       " ('admin', 1317),\n",
       " ('OF', 1315),\n",
       " ('four', 1315),\n",
       " ('FUCK', 1312),\n",
       " ('changed', 1309),\n",
       " ('looking', 1308),\n",
       " ('statement', 1304),\n",
       " ('started', 1299),\n",
       " ('editing.', 1298),\n",
       " ('appears', 1292),\n",
       " ('day', 1291),\n",
       " ('\\n\\nThe', 1291),\n",
       " ('media', 1288),\n",
       " ('discuss', 1285),\n",
       " ('specific', 1281),\n",
       " ('interested', 1278),\n",
       " ('within', 1277),\n",
       " ('group', 1267),\n",
       " ('next', 1267),\n",
       " ('address', 1266),\n",
       " ('SHIT', 1264),\n",
       " ('\"\\n', 1263),\n",
       " ('second', 1259),\n",
       " ('move', 1257),\n",
       " ('recent', 1255),\n",
       " ('PIG', 1254),\n",
       " ('wanted', 1250),\n",
       " ('(UTC)', 1249),\n",
       " ('life', 1246),\n",
       " ('currently', 1243),\n",
       " ('See', 1243),\n",
       " ('full', 1238),\n",
       " ('pretty', 1234),\n",
       " ('world', 1234),\n",
       " ('included', 1230),\n",
       " ('big', 1228),\n",
       " ('taken', 1226),\n",
       " ('3', 1226),\n",
       " ('dont', 1226),\n",
       " ('related', 1225),\n",
       " ('wish', 1222),\n",
       " ('certainly', 1222),\n",
       " (\"there's\", 1213),\n",
       " ('per', 1210),\n",
       " ('words', 1208),\n",
       " ('writing', 1207),\n",
       " ('review', 1207),\n",
       " ('POV', 1205),\n",
       " ('end', 1205),\n",
       " ('American', 1202),\n",
       " ('completely', 1200),\n",
       " ('2005', 1198),\n",
       " ('placed', 1195),\n",
       " ('possible', 1194),\n",
       " ('Not', 1194),\n",
       " ('relevant', 1192),\n",
       " ('issues', 1191),\n",
       " ('during', 1191),\n",
       " ('DIE', 1188),\n",
       " ('website', 1186),\n",
       " ('came', 1185),\n",
       " ('general', 1177),\n",
       " ('wrote', 1175),\n",
       " ('care', 1172),\n",
       " ('especially', 1169),\n",
       " ('love', 1167),\n",
       " ('answer', 1167),\n",
       " (\"wasn't\", 1166),\n",
       " ('=', 1163),\n",
       " ('info', 1162),\n",
       " ('fucking', 1160),\n",
       " ('Thanks.', 1158),\n",
       " ('learn', 1157),\n",
       " ('sentence', 1150),\n",
       " (\"won't\", 1150),\n",
       " ('example', 1147),\n",
       " ('talking', 1145),\n",
       " ('maybe', 1145),\n",
       " ('guidelines', 1145),\n",
       " ('research', 1139),\n",
       " ('thanks', 1138),\n",
       " ('single', 1138),\n",
       " ('times', 1138),\n",
       " ('so,', 1137),\n",
       " ('When', 1135),\n",
       " ('hard', 1129),\n",
       " ('there.', 1123),\n",
       " ('common', 1117),\n",
       " ('order', 1116),\n",
       " ('articles.', 1114),\n",
       " (\"you've\", 1113),\n",
       " ('due', 1110),\n",
       " ('appropriate', 1098),\n",
       " ('template', 1098),\n",
       " ('public', 1097),\n",
       " (\"That's\", 1097),\n",
       " ('looks', 1090),\n",
       " ('asked', 1087),\n",
       " ('year', 1086),\n",
       " ('appreciate', 1086),\n",
       " ('again.', 1084),\n",
       " ('Go', 1082),\n",
       " ('pages,', 1082),\n",
       " ('do.', 1082),\n",
       " ('contributions', 1080),\n",
       " ('notable', 1079),\n",
       " ('took', 1078),\n",
       " ('official', 1077),\n",
       " ('web', 1076),\n",
       " ('working', 1075),\n",
       " ('become', 1073),\n",
       " ('United', 1071),\n",
       " ('nor', 1068),\n",
       " ('picture', 1066),\n",
       " ('hate', 1065),\n",
       " ('noticed', 1065),\n",
       " ('true', 1064),\n",
       " ('fish', 1063),\n",
       " ('claims', 1061),\n",
       " ('facts', 1061),\n",
       " ('perhaps', 1061),\n",
       " ('(or', 1060),\n",
       " ('away', 1056),\n",
       " ('edited', 1055),\n",
       " ('interest', 1055),\n",
       " ('vandalize', 1051),\n",
       " ('political', 1050),\n",
       " ('now.', 1048),\n",
       " ('published', 1045),\n",
       " ('1', 1044),\n",
       " ('tried', 1042),\n",
       " ('TO', 1042),\n",
       " ('deletion,', 1038),\n",
       " ('etc.', 1038),\n",
       " ('–', 1033),\n",
       " ('sorry', 1024),\n",
       " ('deleting', 1018),\n",
       " ('taking', 1018),\n",
       " ('similar', 1014),\n",
       " ('topic', 1013),\n",
       " ('summary', 1013),\n",
       " ('posted', 1012),\n",
       " ('sense', 1010),\n",
       " ('war', 1002),\n",
       " ('Bark', 1002),\n",
       " (\"haven't\", 997),\n",
       " ('reading', 994),\n",
       " ('articles,', 993),\n",
       " ('reverting', 993),\n",
       " ('certain', 992),\n",
       " ('exactly', 990),\n",
       " ('nigger', 989),\n",
       " ('entire', 987),\n",
       " ('almost', 987),\n",
       " ('five', 982),\n",
       " ('way,', 979),\n",
       " ('aware', 978),\n",
       " ('unless', 978),\n",
       " ('report', 976),\n",
       " ('One', 976),\n",
       " ('along', 975),\n",
       " ('...', 975),\n",
       " ('mind', 975),\n",
       " ('likely', 974),\n",
       " ('often', 973),\n",
       " ('states', 971),\n",
       " ('actual', 971),\n",
       " ('Block', 970),\n",
       " ('sort', 967),\n",
       " ('time,', 966),\n",
       " ('course', 966),\n",
       " ('Can', 965),\n",
       " ('Wiki', 965),\n",
       " ('major', 965),\n",
       " ('notability', 964),\n",
       " ('appear', 964),\n",
       " ('everything', 964),\n",
       " ('quote', 962),\n",
       " ('sex', 960),\n",
       " ('everyone', 959),\n",
       " ('later', 956),\n",
       " ('line', 955),\n",
       " ('involved', 953),\n",
       " ('various', 950),\n",
       " (\"article's\", 947),\n",
       " ('attack', 946),\n",
       " ('told', 945),\n",
       " ('particular', 945),\n",
       " ('stuff', 941),\n",
       " ('names', 939),\n",
       " ('again,', 938),\n",
       " ('encyclopedia.', 936),\n",
       " ('lead', 936),\n",
       " ('obviously', 933),\n",
       " ('search', 930),\n",
       " ('decide', 929),\n",
       " ('went', 926),\n",
       " ('contributing', 921),\n",
       " ('IN', 919),\n",
       " ('US', 917),\n",
       " ('stated', 917),\n",
       " ('provided', 916),\n",
       " ('myself', 913),\n",
       " ('thank', 913),\n",
       " ('days', 913),\n",
       " ('saw', 912),\n",
       " ('happy', 911),\n",
       " ('shows', 910),\n",
       " ('sources.', 909),\n",
       " ('generally', 905),\n",
       " ('guess', 904),\n",
       " ('remember', 903),\n",
       " ('nice', 901),\n",
       " ('improve', 900),\n",
       " ('(', 898),\n",
       " ('shit', 896),\n",
       " ('policies', 894),\n",
       " ('previous', 892),\n",
       " ('description', 892),\n",
       " ('follow', 884),\n",
       " ('deletion.', 884),\n",
       " ('removed.', 879),\n",
       " ('useful', 878),\n",
       " ('On', 878),\n",
       " ('according', 877),\n",
       " ('proposed', 877),\n",
       " ('uploaded', 873),\n",
       " ('set', 871),\n",
       " (\"he's\", 870),\n",
       " ('now,', 870),\n",
       " ('comes', 869),\n",
       " ('argument', 869),\n",
       " ('neutral', 868),\n",
       " ('automatically', 867),\n",
       " ('is.', 866),\n",
       " ('proper', 865),\n",
       " ('Wikipedia!', 864),\n",
       " ('moved', 864),\n",
       " ('not.', 863),\n",
       " ('past', 863),\n",
       " ('contact', 863),\n",
       " ('paragraph', 863),\n",
       " ('indicate', 862),\n",
       " ('warning', 858),\n",
       " (\"aren't\", 857),\n",
       " ('vandalism.', 857),\n",
       " ('(talk', 855),\n",
       " ('too.', 855),\n",
       " ('British', 854),\n",
       " (\"wouldn't\", 849),\n",
       " ('you?', 848),\n",
       " ('FUCKING', 847),\n",
       " ('piece', 847),\n",
       " ('although', 845),\n",
       " ('whatever', 845),\n",
       " ('form', 844),\n",
       " ('avoid', 843),\n",
       " ('Here', 843),\n",
       " ('small', 843),\n",
       " ('Are', 842),\n",
       " ('PENIS', 842),\n",
       " ('(which', 842),\n",
       " ('Now', 840),\n",
       " ('case,', 840),\n",
       " ('BALLS', 839),\n",
       " ('creating', 839),\n",
       " ('/', 837),\n",
       " ('terms', 835),\n",
       " ('community', 830),\n",
       " ('?', 829),\n",
       " ('\\nI', 828),\n",
       " ('present', 825),\n",
       " (\"Don't\", 825),\n",
       " ('type', 825),\n",
       " ('Fuck', 824),\n",
       " ('it?', 820),\n",
       " ('4', 819),\n",
       " ('response', 817),\n",
       " ('recently', 817),\n",
       " ('\"\"The', 815),\n",
       " ('allowed', 813),\n",
       " ('all.', 813),\n",
       " ('well.', 812),\n",
       " ('Again,', 812),\n",
       " ('Also', 811),\n",
       " ('cited', 811),\n",
       " ('them,', 809),\n",
       " ('attacks', 809),\n",
       " ('side', 808),\n",
       " ('User', 805),\n",
       " ('test', 804),\n",
       " ('man', 801),\n",
       " ('2006', 801),\n",
       " ('Good', 800),\n",
       " ('REDIRECT', 798),\n",
       " ('2004', 797),\n",
       " ('(I', 797),\n",
       " ('attempt', 796),\n",
       " ('U', 796),\n",
       " ('deleted.', 794),\n",
       " ('short', 794),\n",
       " ('dispute', 792),\n",
       " ('explaining', 791),\n",
       " ('definition', 791),\n",
       " ('live', 791),\n",
       " ('stupid', 789),\n",
       " ('enjoy', 788),\n",
       " ('itself', 788),\n",
       " ('longer', 786),\n",
       " ('open', 784),\n",
       " ('guys', 782),\n",
       " ('cite', 781),\n",
       " ('Take', 781),\n",
       " ('accept', 781),\n",
       " ('tildes', 780),\n",
       " ('encyclopedia', 779),\n",
       " ('By', 778),\n",
       " ('example,', 776),\n",
       " ('FAT', 776),\n",
       " ('guy', 776),\n",
       " ('obvious', 775),\n",
       " ('information.', 774),\n",
       " ('ass.', 774),\n",
       " ('project', 774),\n",
       " ('admins', 774),\n",
       " ('explanation', 773),\n",
       " ('high', 773),\n",
       " ('\"\\nI', 771),\n",
       " ('WP', 771),\n",
       " ('therefore', 770),\n",
       " ('all,', 770),\n",
       " ('goes', 769),\n",
       " ('knowledge', 767),\n",
       " (\"You're\", 764),\n",
       " ('\\n\\nYou', 764),\n",
       " ('Even', 764),\n",
       " ('Since', 763),\n",
       " ('5', 763),\n",
       " ('false', 763),\n",
       " ('file', 762),\n",
       " ('faith', 761),\n",
       " ('May', 761),\n",
       " ('result', 759),\n",
       " ('problems', 757),\n",
       " ('living', 756),\n",
       " ('July', 755),\n",
       " ('Some', 754),\n",
       " ('username', 754),\n",
       " ('edits,', 753),\n",
       " ('large', 753),\n",
       " (\"shouldn't\", 752),\n",
       " ('lack', 751),\n",
       " ('usually', 751),\n",
       " ('At', 749),\n",
       " ('HATE', 747),\n",
       " ('country', 747),\n",
       " ('accepted', 745),\n",
       " ('refer', 745),\n",
       " ('separate', 743),\n",
       " ('government', 742),\n",
       " ('points', 740),\n",
       " ('tags', 740),\n",
       " ('Maybe', 739),\n",
       " ('\\n\\nIf', 738),\n",
       " ('Perhaps', 736),\n",
       " ('supposed', 735),\n",
       " ('upon', 733),\n",
       " ('entry', 733),\n",
       " ('external', 731),\n",
       " ('reasons', 731),\n",
       " ('These', 730),\n",
       " ('game', 728),\n",
       " ('school', 728),\n",
       " ('2008', 728),\n",
       " ('multiple', 728),\n",
       " ('source.', 727),\n",
       " ('gets', 726),\n",
       " ('content,', 726),\n",
       " ('play', 726),\n",
       " ('heard', 725),\n",
       " ('one.', 724),\n",
       " ('news', 722),\n",
       " ('tagged', 720),\n",
       " ('disagree', 717),\n",
       " ('WANKER', 716),\n",
       " ('Of', 716),\n",
       " ('power', 716),\n",
       " ('valid', 715),\n",
       " ('members', 712),\n",
       " ('John', 711),\n",
       " ('couple', 711),\n",
       " ('below', 710),\n",
       " ('fact,', 709),\n",
       " ('calling', 707),\n",
       " ('\"\\n\\nI', 707),\n",
       " ('copy', 706),\n",
       " ('Jews', 705),\n",
       " ('asking', 703),\n",
       " ('yourself,', 702),\n",
       " ('books', 701),\n",
       " ('existing', 701),\n",
       " ('up.', 701),\n",
       " ('date', 699),\n",
       " ('described', 699),\n",
       " ('assume', 698),\n",
       " ('2007', 697),\n",
       " ('produce', 697),\n",
       " ('shall', 697),\n",
       " ('rest', 697),\n",
       " ('deal', 695),\n",
       " ('\\n\\nHi', 694),\n",
       " ('banned', 692),\n",
       " ('contribs)', 690),\n",
       " ('interesting', 690),\n",
       " ('however,', 690),\n",
       " ('not,', 689),\n",
       " ('human', 687),\n",
       " ('date.', 686),\n",
       " ('prove', 686),\n",
       " ('Or', 683),\n",
       " ('meant', 683),\n",
       " ('historical', 682),\n",
       " ('sources,', 680),\n",
       " ('addition', 680),\n",
       " ('status', 680),\n",
       " ('rule', 679),\n",
       " ('available', 678),\n",
       " ('simple', 677),\n",
       " ('hi', 676),\n",
       " ('administrator', 674),\n",
       " ('fix', 670),\n",
       " ('themselves', 669),\n",
       " ('ones', 669),\n",
       " ('August', 669),\n",
       " ('pillars', 669),\n",
       " ('bring', 667),\n",
       " ('statements', 666),\n",
       " ('position', 666),\n",
       " ('indeed', 665),\n",
       " ('process', 665),\n",
       " ('ban', 664),\n",
       " ('OH', 664),\n",
       " ('doubt', 663),\n",
       " ('Then', 662),\n",
       " ('giving', 662),\n",
       " (\"they're\", 662),\n",
       " ('citation', 660),\n",
       " ('\\n\\nThis', 660),\n",
       " ('know,', 659),\n",
       " ('blocking', 659),\n",
       " ('(as', 659),\n",
       " ('legal', 658),\n",
       " ('system', 658),\n",
       " ('help,', 654),\n",
       " ('there,', 654),\n",
       " ('meaning', 654),\n",
       " ('out.', 654),\n",
       " ('allow', 653),\n",
       " ('hours', 653),\n",
       " ('film', 652),\n",
       " ('contribute', 652),\n",
       " ('primary', 651),\n",
       " ('While', 651),\n",
       " ('10', 651),\n",
       " ('(the', 650),\n",
       " ('towards', 650),\n",
       " ('views', 650),\n",
       " ('uses', 650),\n",
       " ('needed', 650),\n",
       " ('cause', 649),\n",
       " ('soon', 646),\n",
       " ('March', 646),\n",
       " ('sandbox', 646),\n",
       " ('edits.', 646),\n",
       " ('author', 645),\n",
       " ('changing', 645),\n",
       " ('future', 645),\n",
       " ('among', 641),\n",
       " ('attention', 641),\n",
       " ('(see', 640),\n",
       " ('serious', 640),\n",
       " ('complete', 639),\n",
       " ('gave', 638),\n",
       " ('respect', 637),\n",
       " ('FAG', 637),\n",
       " ('directly', 636),\n",
       " ('24', 636),\n",
       " ('reported', 636),\n",
       " ('particularly', 636),\n",
       " ('contest', 635),\n",
       " ('\\n\\nPlease', 634),\n",
       " ('information,', 633),\n",
       " ('people,', 631),\n",
       " ('stay', 630),\n",
       " ('happened', 629),\n",
       " ('gay', 629),\n",
       " ('Did', 629),\n",
       " ('close', 629),\n",
       " ('moron', 628),\n",
       " ('week', 628),\n",
       " ('THIS', 627),\n",
       " ('section.', 625),\n",
       " ('NOES', 625),\n",
       " (\"you'll\", 625),\n",
       " ('actions', 625),\n",
       " ('email', 622),\n",
       " ('JEW', 621),\n",
       " ('yourselfgo', 621),\n",
       " ('requesting', 621),\n",
       " ('work.', 620),\n",
       " ('freezer', 619),\n",
       " ('coming', 619),\n",
       " ('Have', 619),\n",
       " ('however', 617),\n",
       " ('early', 617),\n",
       " ('totally', 617),\n",
       " ('family', 616),\n",
       " ('together', 616),\n",
       " ('reply', 612),\n",
       " ('so.', 611),\n",
       " ('CUNT', 611),\n",
       " ('Feel', 611),\n",
       " ('speak', 610),\n",
       " ('release', 610),\n",
       " ('run', 610),\n",
       " ('German', 610),\n",
       " ('merely', 610),\n",
       " ('thinking', 610),\n",
       " ...]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "occurences_counter.most_common()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "occurences_vec = [a[1] for a in occurences_counter.most_common()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.text.Text at 0x7f4768aea5f8>"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYcAAAEKCAYAAAD5MJl4AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xu8XFV99/HP9yQkIQkQrgkkkgRoCEKotRJ8UGSQCqjV\nVKVifHF5aC2t+qDWqgHtYw5SkRZRi1RsC02lGKOtgsYLUjEHDBSIcolCQiIiEALhDkauJr/+sdch\n+8zlZJ+cM7NnMt/363VeZ/bae9b67bX3zJq11p49igjMzMzyesoOwMzM2o8bBzMzq+HGwczMarhx\nMDOzGm4czMyshhsHMzOr4cbBBiXpHkmvb7DutZJWtTqmdiXpF5JeN8j6ZZL+rGBeR0m6f+SiK07S\nWZL+ZZD1p0r6yQiW9xtJM0Yqv2HE0fBc70ajyw7AOldELAcO2tp2khYC+0fEKc2PqjwRcUj/4xHa\n51K+hBQRn+l/LGk6cA8wOiI25zcbwfJ2Gqm8bOS451AiSaPKjqHTuQ5HVp36FFlDoBLCsRJ1feMg\naXbq7j8h6eeS3pJbN07SBZJ+ndZfJ2lsWvdaSden9HslnZLSBwwdVHfBJW2W9D5Ja4A1uRiulvSY\npFWS/jS3/SJJF0n6rqSnJf2PpJm59QfnnvugpDNTuiSdKemXkh6RtETSpLRurKT/kPRoiv8mSXsO\nUk1/IOn2tO3XJI1J+QwY+pC0QNK6FOcqSUdLOg74OHBiGj64NW27t6Rvp7jXSHpPVb1/RdLjku6Q\n9NGqcu6R9DFJtwMbJfWksn+Zyv6FpD+pOgbLJX0u7cMvJf2flH6fpIf6j1+d86MiaWVu+UeSbsot\n/0TSW3Nxvb7RPiczUixPS7pK0m6D1Hs+jsHO090kLZX0VDqW51Sdc19I+/mUpBWSXptbt1DSf6bz\n4Ung1JR2Wdrk2vT/yRTz4VueqvPTMbpb0vG5PJelGK5P+//tFOPluRj3zW2/WdJ+6XHD11xVfeye\n9vmJdA5dm1s3TdI3JT2czv0LU/p+kq5J5/3DKZ6dG9S31OD10zUiomv/yIbV1gIL0uOjgaeB30vr\n/wn4MTCF7JPTq4EdgJel7d4JjAJ2BQ5Nz1kG/FmujFOB63LLm4EfApOAscB44D7glFTGK4BHgIPS\n9ouAR4E/JGvMLwcWp3UTgfXAh4AxwATgsLTuQ8ANwN4p5otzzzsd+HYqX8AfABMb1NE9wI3A5BTz\nncDpad1RwH3p8ay0H5PT8r7AzPR4IXBZVb7XAl9Msf0+8DBwdFp3XqrHnYF9gNv7y8nFdEtaNzal\nvSNX9p8CG3PLpwIv5Or4HODeXPlvSMdzfJ39Hwv8FtgtHesHgXWprscBzwCTcnG9fpB9XkZ2vu2f\n8l0GnNug3vN1u7XzdAmwOOV5UDoO+XPu3enY9QB/nfZhTC7O54G3pOVx+diB6cAmQFXn9AvAn6X6\n/Cvggar9XAPMAHYC7gBWp7h7gK8Al+a23wTsN9hrrk79nAt8KeU3CnhNSu8BbgM+m/ZlDHBEWrc/\ncEyqw92BPuBzVedV//Fr+Prplr/SAyh15+G1wPqqtMXAJ9OJ+QxwSJ3nnQl8s0GeRRqHo3LL7wSu\nrcrjy8D/T48XAf+SW/dG4M70eD7wswZx3El6s03Le6cXdA9wGrAcmFOgju4B5ueW/x74UnqcfwPb\nH3io/8VXlceAN0pgGvAiuTfj9GL/t/T4buCPcuv+nNrG4dStxH0rW97wTgXuyq07JL0h7ZFLe5TU\nwNfJ61rgT4DDyRr2JcCxQAW4rSqurTUOH88tvxf4foMy83V75CDnaU86rgfk1p2TP+fq5P14/7FP\ncfY1Ol5saRx6qs7pNbnlHdN5vVduP8/Krf8s8L3c8h8Dt1S9JvZjkNdcnX04G7iCbF4nn/5qYEM+\n3kHymEfu9VN1/Bq+fraW7/by1+3DSvsA1VeE3AtMBfYg++TxqzrPexnZG9i2Wpd7PB14deqePy7p\nCbJPepNz2zyUe/wMWY8BsjfZRnFMB67oz5fsZH8x5fsfpDc5ZcNA52nwsfsNDcp/SUTcTfZpqxfY\nIGmxpCkN8tsHeDwinsml9dd7//p8HdW7aie/HkmnSLo1DTM8ARxMdgzr7cOzKeZHq9Jq9iu5juxT\n7+vIPm32kTUMR7Fl2KWoRsdyMHvT+Dzdk+yTcMP6kvQ3ku7M1c3ODKybbbkq6qX9iIhn08P8vlTX\nd/Vyvf3eg6z3U+81V+0fyM79q9PQz4KU/jLg3hg4eQ6ApD2VDYuuS0NolzOwHvIGe/10hW5vHNaT\nnUx5+wIPkH2SfI7sE3G1+4EDGuT5W7Khon713iCjKq++iNgt/e0aETtHxP8rEP9gcdwHvLEq3wkR\n8WBE/C4izomIg4EjgLeQDbkMS0QsiYgjyV5YkPUyoPbKlvXAbpIm5NL66x2yYY9pVetqiut/kMav\n/wV4X9rPXcmGMkZqEvVassbgyPT4OrKG4XU0bhyq93k4BjtPHwF+x8D6emlbSUcCHwNOyNXN0wys\nm8FiHcn92JrBXnMDRMRvI+IjEbE/2fn7YUlHk70m9pVU773tM2S9lEMiYhJwEo3PkYavn23Yr47U\n7Y3DTcBvlU1ujpZUIevyfi2yvuS/AZ9TNnnaI+nVknYAvgocI+kESaPSZNvvpzxvA94uaUdJB5AN\niQzmu8AsSSelGHaQ9CpJBxaI/7vAZEkfkDRG0kRJc9O6fwbO7Z/4S5+a+idOK5IOSS+gjWSfiDYV\nq7L6JM1SNgE9hqz7/Wwuzw1kE7ECiIh1ZOO5n1E2OX4oWT1dnrb/BnCWpEmSpgLv30rxE8he9I+m\n43Qa2dDRoCEPYfduAA4E5gI3R8SdZA3g4WQNRT0D9nmYBjtPNwPfBHrTOTebgQ39RLLj+1g6Rz5J\nNg9Q1CNkdbvVN+zhSq+5RdR/zQ0g6c2S+mPaSNZAbgJuJvtwcZ6k8en8OiJtt1Pa9ul0Xn10kHAa\nvn66RVMahzTT/3eSLpR0cjPKGAkR8SLwVuBNZJ9aLgJOjoi1aZO/AX4OrAAeI5so7YmI+9NzPkI2\nfnsrcGh6zufJXowPkZ3o/W94LxVbFcNGsvHrd5F9Qlyfyqm5QqNO/BvJJlPfmspbQ/YJF+AfySad\nr5b0FNkbXH/DMQX4L+Apsk/Yy+rEWTfeQYxNcT+S9mFPsit2AP6T7M34MUk/TWnvBmambb9JNsfy\n47TuU2Sfiu8Brk7Pf75RTBGxCriAbOL8IbIhpeVbibd6vxruZxr++hnwi4j4XUr+H+DXVUNT+Tzq\n7fM2fQovcJ6eQTbh/CDZZO9ittTXD4GryM6Ne8iGsgoPI6Uho08D16chlrmNNm3wuFAxuccfoc5r\nrs5zfg/4kaTfANcD/xQR16XG8i1p/X1k+/rO9JyzyS7seBJYSnbeNYpjsNdPV1CabBnZTLPLCOeR\nHdzvRcSyES/EuoakvwJOjIijy46lE0g6j+xKrdPKjsU6V6Geg6RLJW1Q7nrvlH68pNXKrlNfkFt1\nIHBDRHwEeN8IxmtdQNIUSUekHuiBZD24b5UdV7uSdKCkOenxXLIhOteXDUvRYaVFwHH5hDRefVFK\nPxiYn8Y7Ibty4on0+HeYDc0YsjHfp4EfkV2yeHGpEbW3nYBvSdpIdpnt+RGxtOSYrMMVurdSRCxX\ndo+VvLnA2oi4F0DSErKhpNVkn1q+mK6UaDRhZ1ZXRNwHzCk7jk4RET8lG2M3GzHDufHeVAZObK0j\nTdikSaz31HuSmZm1v+E0DvUu0Ss8uy2plddPm5ltNyKi6TdCHM6lrOsY+OWkaWSXJRa2cOFCli1b\nVvrXxPN/CxcuLD0Gx+SYujEuxzT437Jly1i4cOEw3rKHZiiNgxjYW1gBHCBpevri07uA7wyl8N7e\nXiqVylCeYmbWlSqVCr29vS0rr+ilrIvJvgQyS9mtf0+LiE1kX765muyLVEsi+zJSYb29vfT19Q0x\nZDOz7tPX19fSxqHo1UrvbpD+A+AH21p4K3e0qHbsyTimYhxTce0Yl2MaXKVSoVKpcPbZZ7ekvKZ8\nQ7pQwVIsXLjwpR02M7PG+vr66Ovr4+yzzyZaMCFdauNQVtlmZp1KUksah26/K6uZmdVRauPgCWkz\ns2JaPSHtYSUzsw7iYSUzMyuNh5XMzDqAh5XMzKwhDyuZmVlpPKxkZtYBPKxkZmYNeVjJzMxK48bB\nzMxquHEwM7ManpA2M+sAnpA2M7OGPCFtZmalceNgZmY13DiYmVkNNw5mZlbDjYOZmdXwpaxmZh2g\nqy5l3bw5UNMvyDIz2374UlYzMytNqY2DvwNnZtae3HMwM7Ma7jmYmVmNpjQOko6SdJ2kiyW9rhll\nmJlZ8zSr5xDAb4CxwLomlWFmZk1SqHGQdKmkDZJWVqUfL2m1pDWSFvSnR8R1EfFm4EzgU43y9bCS\nmVl7KtpzWAQcl0+Q1ANclNIPBuZLml31vCeBMcMN0szMWmt0kY0iYrmk6VXJc4G1EXEvgKQlwDxg\ntaS3kTUau5A1IA3y3aaYzcysyQo1Dg1MBe7PLa8jazCIiCuAK7aWwac+1cuoUdnjSqVCpVIZRjhm\nZtufvr6+Um4zVPj2GannsDQiDk3LJwDHRsTpafkk4LCI+GDB/OL554MxHnQyMyusE26fsQ7YN7c8\nDVg/lAzOOcc33jMzK6LVN94bSuOg9NdvBXCApOmSxgDvAr4zlMI952Bm1p4KDStJWgxUgN2BDcDC\niFgk6Y3AF8gamUsj4rzCBUvx3HPB2LHbFLeZWVdq1bBSqbfsduNgZjY0nTDnMGyeczAzK6arfuzn\n2WeDceNKKd7MrCO552BmZi/pqp7DM88EO+5YSvFmZh2pK3oOvpTVzKw9ldo4fPrTHlYyMyuiq4aV\nfvvbYPz4Uoo3M+tIHlYyM7PSlNo4mJlZe/Kcg5lZB+iqOYff/CaYOLGU4s3MOpLnHMzMrDSeczAz\nsxqlNg7nnus5BzOzIrpqzuGpp4Kddy6leDOzjtQVcw5mZtaePCFtZmY13HMwM7Ma7jmYmVkN9xzM\nzKxGqY3DZz7jS1nNzIroqktZH3882HXXUoo3M+tIXXEpq+cczMzak+cczMyshhsHMzOr4WElMzOr\n0bTGQdJ4ST+V9KZmlWFmZs3RzJ7DAuDrg23gnoOZWXsq1DhIulTSBkkrq9KPl7Ra0hpJC3LpxwB3\nAg8DTb/kyszMRlbRnsMi4Lh8gqQe4KKUfjAwX9LstPpo4HDg3cB7GmXqnoOZWXsaXWSjiFguaXpV\n8lxgbUTcCyBpCTAPWB0Rf5vSTgEeHcF4zcysBQo1Dg1MBe7PLa8jazBeEhGXDZbBP/xDLxMmZI8r\nlQqVSmUY4ZiZbX/6+vpKuc1Q4dtnpJ7D0og4NC2fABwbEaen5ZOAwyLigwXziw0bgr322rbAzcy6\nUSfcPmMdsG9ueRqwfigZnH++b7xnZlZE2954T9IMsp7DnLQ8CrgLOAZ4ELgZmB8RqwrmFw89FEye\nvA1Rm5l1qbbqOUhaDNwAzJJ0n6TTImITcAZwNXAHsKRow9DPPQczs2Latucw4gVL8eCDwZQppRRv\nZtaR2qrn0Cyf/ax7DmZmRXRVz2H9+mDvvUsp3sysI7nnYGZmL3HPwczMGuqKnoPvrWRm1p5KbRwu\nuMDDSmZmRXTVsNK6dcHUqaUUb2bWkbpiWMnMzNqT5xzMzKyG5xzMzDpAV805/OpXwcyZpRRvZtaR\numLOYdOmMks3M7NG3DiYmVmNUhuHCy/0nIOZWRFdNeewcmUwZ04pxZuZdSTPOZiZWWncOJiZWQ03\nDmZmVsONg5mZ1Si1cdi8uczSzcyskdFlFn7JJb28+GKFSqVSZhhmZm2vr6+vpZf+l3op6zXXBK9/\nfSnFm5l1pK64lPXFF8ss3czMGim1cXjhhTJLNzOzRtw4mJlZDTcOZmZWoylXK0maDXwQ2B34cUR8\nud52zz/fjNLNzGy4mtJziIjVEfFe4ETgiEbbuedgZtaeCjUOki6VtEHSyqr04yWtlrRG0oKqdW8B\nvgt8v1G+bhzMzNpT0Z7DIuC4fIKkHuCilH4wMD8NJwEQEUsj4s3ASY0ydeNgZtaeCs05RMRySdOr\nkucCayPiXgBJS4B5wGpJRwFvB8YC32uUr+cczMza03AmpKcC9+eW15E1GETEtcC1W8vghz/sfamB\nqFR8Gw0zs2qtvm1Gv8K3z0g9h6URcWhaPgE4NiJOT8snAYdFxAcL5hcf/3jw6U9vW+BmZt2oE26f\nsQ7YN7c8DVg/lAyWLfNvSJuZFdG2vyEtaQZZz2FOWh4F3AUcAzwI3AzMj4hVBfOLM84ILrxwG6I2\nM+tSbdVzkLQYuAGYJek+SadFxCbgDOBq4A5gSdGGod/117vnYGZWRNv2HEa8YClOPjm47LJSijcz\n60ht1XNolhUr3HMwMyuiq3oOb3hDcPXVpRRvZtaRuqLnsGqVew5mZkV0Vc/h0EOD228vpXgzs47U\nFT2HjRvLLN3MzBoptXHYsMHDSmZmRXTVsNKECeHeg5nZEHTFsNIzz8DmzWVGYGZm9ZTaOIwblzUQ\nZmbWXkptHHp6ernqqr4yQzAz6whdNecwa1Zw5ZVw0EGlhGBm1nG6Ys5h0iR48skyIzAzs3pKbRx2\n2w2eeKLMCMzMrJ5SG4c99oBHHikzAjMzq2c4vyE9bGvX9rLTThVOPbVSZhhmZm2v1b8lXeqE9Kc+\nFTz3HP4daTOzgrpiQnqvvTysZGbWjkptHKZMgfXry4zAzMzqKbVxmDoVHnigzAjMzKye0hsH9xzM\nzNpPqVcrXXxxL489VuGFFyqMGVNmJGZm7a2rrlaKCGbMgGuugf33LyUMM7OO0hVXKwHMmAG//nXZ\nUZiZWV7pjcP06W4czMzaTemNw377wT33lB2FmZnlld44zJgBd91VdhRmZpbXtKuVJM0D3gzsBPxb\nRPx3ve322Qduu61ZUZiZ2bZo+tVKkiYB50fEX1SlR0TwwAMwbVr2W9Jq+vy7mVlna7urlSRdKmmD\npJVV6cdLWi1pjaQFdZ76t8A/Ncp3n32y/489VjQSMzNrtqHMOSwCjssnSOoBLkrpBwPzJc3OrT8P\n+H5ENBw4kmDmTFi5stEWZmbWaoUbh4hYDlT/bttcYG1E3BsRLwJLgHkAks4AjgFOkHT6YHkffDDc\neOOQ4jYzsyYa7oT0VOD+3PI6sgaDiPgi8MXBntzb2wvAxo3Q11fh4x+vDDMcM7PtS6tvm9FvSBPS\nkqYDSyPi0LR8AnBsRJyelk8CDouIDxbIK/rLvuIKOOccuOWWbdgDM7Mu0qoJ6eH2HNYB++aWpwGF\n77Pa29tLpVLhsMMq3HrrMCMxM9uOtfWN9yTNIOs5zEnLo4C7yOYWHgRuBuZHxKoCeb3Uc4iAnp7s\ny3CzZg11F8zMukc7Xsq6GLgBmCXpPkmnRcQm4AzgauAOYEmRhqFfb28vfX19SHDEEfCjHw01fDOz\n7tDX1/fSPG0rlH7L7n4f+Qg8+SRcckkp4ZiZdYS26zk0Q3/PAeA1r4GrriozGjOz9tW1PYdnnoEJ\nE+Dpp2GnnUoJycys7XVFzyFv/Pjs1+CuuabsSMzMrG2GlQCOPBK++93y4jEza1ddO6wE8PWvw4IF\n/mU4M7NGWjWs1FaNw8aN2XzDY4/BbruVEpaZWVvrijmH6mGliROznw1dsqS8mMzM2lFXDytBNqx0\n552wdGkJQZmZtbmuHFYCWL0aDjoInnsOxo4tITAzszbWlcNKALNnZ78O981vlhOTmVk76vphJchu\npbFiBVx7bYuDMjNrc107rATZpawzZ8Ljj8Ouu7Y2LjOzdtYVw0qNzJiR/XTohReWHYmZWXdqy8YB\n4C/+Av71X8uOwsysO7XdhHS/00+HBx6AG25obUxmZu3IE9I5J5yQzTv8+MctCsrMrM119YR0vzVr\n4MAD4aGHYPLkFgVmZtbG3DgkL385HHIIfOMbLQjKzKzNuXFIrrsOjjrKvQczM3DjMMB++8ERR8Dl\nlzc5KDOzNtcV33MY7GqlvK98Bb76VfjFL5ofk5lZO/LVSg0cdxysXw8rV4Ka3maambWnrug5DMWX\nv5z1HL761bIjMTPb/nVM4zBzJnzgA/D+98PDD5cdjZnZ9q1jhpUAnnwSZs2Co4/Ofi3Ow0tm1m08\nrFTHpEnwta9l33n4yU/KjsbMbPvVlJ6DpJnAJ4CdI+KdDbYZcs+h34c/nP0Y0C23wO67DyNQM7MO\ns118z0HSN5rRODz7LBx2GETAHXcMK0Qzs47SVsNKki6VtEHSyqr04yWtlrRG0oLmhFhrxx3h+uth\n7Vp4wxtaVaqZWfcoOuewCDgunyCpB7gopR8MzJc0u+p5TWvddtklaxyWLct+FGjjxmaVZGbWfQo1\nDhGxHHiiKnkusDYi7o2IF4ElwDwASbtJuhh4RTN7FNOnw7nnwgUXwI03NqsUM7PuM3oYz50K3J9b\nXkfWYBARjwPvHUbehX3sY9m3pt/5Tpg3DxYtakWpZmbbt+E0DvWGjIY0w5y/T0ilUqFSqWxTIJdc\nks1BvPe98LOfwQEHZMNOZmadrq+vr9A96EbacBqHdcC+ueVpwPqhZjKcRqHfuHHwylfCHnvA294G\n73gHfP7zw8rSzKwt9L9HtrqRKHwpq6QZwNKImJOWRwF3AccADwI3A/MjYlXB/Lb5UtbB/Pu/Z7f2\n/tCHsjmJOXNGvAgzs9K026Wsi4EbgFmS7pN0WkRsAs4ArgbuAJYUbRj6Fb1l91C86lXZpa7nnw8n\nnzyiWZuZlca37B4hd98NlQr0tz177gk779y04szMWqJVPYfhzDkMW29v74jMOdQzeXI2KX3ssfDM\nM1mPYunSES/GzKwl2nbOYcQLbnLPIe8nP4GzzoLly1tSnJlZ03RFz6FVdtoJ7rwT3v3uLWnz5sGJ\nJ5YXk5lZO9tuh5XyDjkE/vmf4cUXs+UbboArr3TjYGadw8NKLXDlldk3qb/97VKKNzPbZh5WaqIJ\nE7Lewx//8cD0cePgsstg/Phy4jIzaxddMaxU7XWvg698BTZvHpj+53+e/T71jBktDcfMbKs8rFSi\n2bOzIafZ1TceNzNrEx5WKsG4cXD//YPftG/PPWG0a83MtnNdOazUyJw5cMopjddv3Agf/Sh88pOt\ni8nMDDys1NY++1l48MHsx4XMzMrQVjfes8yYMfDCC2VHYWbWfG4chmDsWDcOZtYdPLU6BGPGwC23\nbNuwkgQnnQR77TXycZmZjTRPSA/BUUfBHXfA+iH/3h1873uw994wf/7Ix2Vm2z9PSG+nTj45u324\nf4DIzIbDE9LbmR122HLjPzOzdufGoUVGj4bf/a7sKMzMinHj0CKjR7vnYGadw41Di+ywg3sOZtY5\nPCHdIp/4BHzxizBxYtmRZMaMgRUrsntFmVnn6Iob73XapazD0dsL739/2VFsceSR8MQTbhzMOoUv\nZbWWOOgg+Na3sv9m1jl8Kas1VU9P7Y8dmZn1c+PQpUaNgk2byo7CzNqVG4cu1dPjxsHMGnPj0KVG\njfKwkpk11pSrlSSNB74EPA9cGxGLm1GObTv3HMxsMM3qObwd+M+I+EvgrU0qoylaealYUc2Iabg9\nh26pp+Fqx5igPeNyTO2lUOMg6VJJGyStrEo/XtJqSWskLcitmgbcnx531OfTdjwZmhHTcHsO3VJP\nw9WOMUF7xuWY2kvRnsMi4Lh8gqQe4KKUfjAwX9LstPp+sgYCoOnX49rQ+WolMxtMoTmHiFguaXpV\n8lxgbUTcCyBpCTAPWA1cAVwk6c3A0hGM10bI6NGwYAHssce2Pf+uu+BnPxvZmIbLMRXXjnFtbzFd\ncAHMmjWy8bRS4W9Ip8ZhaUQcmpbfARwXEaen5ZOAuRHxgYL5+evRZmbboN3vrVQvuMJv+K3YOTMz\n2zbDuVppHbBvbnkasA2/rmxmZu1mKI2DGNhbWAEcIGm6pDHAu4DvjGRwZmZWkojY6h+wmKxX8Dxw\nH3BaSn8jcBewFjizSF7peceTTVyvARYUfd5Q/oBfA7cDtwI3p7RdgatTzD8Edsltf2Haj9uAV+TS\nT01x3gWckkt/JbAyrftCgxguBTYAK3NpTY9hK2XUi2khWU/wlvR3fG7dWSmmVcCxWzuGwAzgxlT2\n14DRKX0MsCTl9T/AvrnnTAN+DNwJ/Bz4QNl1VSemM9qkrsYCN5Gd1z8HFg4jr5GKt1FMi4BfpfRb\ngENbea6n9T2p7O+UXU9VMd2ai+nfy66nuu9fzXhjHrTArGJ+CUwHdkg7PbsJ5fwK2LUq7e+Bj6XH\nC4Dz0uM3At9Ljw8HbsxV6N3ALsCk/sdp3U1kE/AA3yebnK+O4bXAKxj4Rtz0GBqVMUhMC4EP14n/\noHTCjk4n/C/Jeo8NjyHwdeBP0+OLgb9Mj98LfCk9PhFYkitnSv+JD0xMJ/DsMutqkJhKrauUNj79\nH0X2JnT4UPMCXj5S8Q4S0yLg7XXqqiXnekr7a+BytrwRl1pPDWJaBLytzHqq+x46Em/EQ/kDXg38\nILd8Jk3oPQD3ALtXpa0GJude/KvS4y8DJ+a2WwVMJhsquziXfnE6caYAd+bSB2xXVeZ0Br4RNz2G\nOmWs3kpMC4G/qRP7gGMD/CCdpA2PIfAI0FN9rIGrgMPT41HAI4McuyuBP2qHuqqK6Zh2qitgPPBT\nssvKHy6Y18MjGO9Vg8R0GNmb3jvqbNOS40fW+/tvoMKWN+Kidd6UemoQU6n11OivjBvvTWXLt6ch\n66JPbUI5AfxQ0gpJ70lpkyNiA0BEPATstZWYqtMfyKWvq7N9EXu1IIbq/Szye2/vl3SbpEsk7dIg\npnzZNbFK2h14IiI259Or84qITcCTknarDkLSDLKezY119qOUusrFdFNKKrWuJPVIuhV4iOyN5m7g\nyYJ5PZXyGol492kUU0SsSKv+LtXVBZJ2aFBXzTp+nwc+SrqKcoh13pR6qo4pp8x6qquMxmFYl8AO\nwRER8SrgTWQv5iMHKac6JqVtG8XajH0oM4YvAftHxCvIXtwXNIhpa2VXX7SQj6nR/m1JkCYC/wV8\nMCI2Vq9sqSDtAAACl0lEQVSvem69vEa8rurEVHpdRcTmiPgDsk+hc8mGtOqVsa1xDTXempgkvZxs\nHvIgsl7E7mTDGfViGvHjl76AuyEibss9fyh1vrWyh1xPDWKCEutpMGU0Di25BDa1jkTEI2RDAnOB\nDZImA0iaQtYV74/pZXViahRro+2LaEUMDzUoo66IeCRSfxP4V7K6GnJMEfEoMCndWqU6ppfykjQK\n2DkinujPQNJosjfh/4iIb6fkUuuqXkztUFf9IuJp4FqyoYuiee2S8hrJeOvFdHzuk+qLZEMn21RX\ng2wPjY/fa4C3SvoV2aTw64EvALuUWE81MUm6rOR6amxr404j/Uc2ntc/kTOGbCLnoBEuYzwwMT2e\nAFwPHEs2KdM/LngmWyYf38SWiZ9XU3/ip//xpLTupnQQRTbxc3yDWGYAP88tNz2GqjLqTdJVxzQl\n9/ivgcXpcf9k3BhgJlsm4+odw/xk3ImxZSz0r9Lj97Flwu9d1E6yXgZ8riqt1LpqEFOpdQXswZbJ\nxx2B61J9DCmvEY63UUxTUprIhlPObfW5ntKPYuCEdCn1NEhMbVFPNTGO5Jty0T+yS8CGfAnsEPKf\nmQ5W/6V1Z6b03YAfpbL/u79C07qL0sG+HXhlLv3/pjjXMPCSsT9Mea8F/rFBHDWXAKeD2dQYtrKf\n9WK6jOzyt9vIelmTc9uflWKqdxlfzTFMdX9TivXrwA4pfSzwjbT9jcCM3HNeQ3b33v5jdkvKv+nH\nq1EZg8RUdl3NSbHcluL4xDDyGql4G8V0TTo+K1O9jW/luZ7bJv9GXFo9DRJTW9RT9V/heyuZmVn3\n8M+EmplZDTcOZmZWw42DmZnVcONgZmY13DiYmVkNNw5mZlbDjYOZmdVw42BmZjX+F3cVHPC8dfzF\nAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f47695940f0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.semilogy(occurences_vec)\n",
    "plt.title('occurences histogram with logarithmic scale')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Now that we have plotted the words occurences, we are happy so we will divide our data to X and Y (X==samples, Y==ground truth) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "class_list = ['toxic','severe_toxic', 'obscene', 'threat','insult','identity_hate']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "Y = train_data[class_list].as_matrix()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X = list(train_data['comment_text'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1, 0, 0, 0, 0, 0],\n",
       "       [0, 0, 0, 0, 0, 0],\n",
       "       [0, 0, 0, 0, 0, 0],\n",
       "       ..., \n",
       "       [0, 0, 0, 0, 0, 0],\n",
       "       [0, 0, 0, 0, 0, 0],\n",
       "       [0, 0, 0, 0, 0, 0]])"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "train_percent = 0.7\n",
    "val_percent = 0.2\n",
    "test_percent = 0.1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "total_samples_num = len(Y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "95851"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "total_samples_num"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "train_sample_num = int(total_samples_num * train_percent)\n",
    "val_sample_num = int(total_samples_num * val_percent)\n",
    "test_sample_num = int(total_samples_num * test_percent)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "67095 19170 9585\n"
     ]
    }
   ],
   "source": [
    "print (train_sample_num,val_sample_num,test_sample_num)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "last_train_index = train_sample_num \n",
    "last_val_index = last_train_index + val_sample_num\n",
    "first_test_index = last_val_index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "train_X = X[:last_train_index]\n",
    "val_X = X[last_train_index:last_val_index]\n",
    "test_X = X[first_test_index :]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "train_Y = Y[:last_train_index]\n",
    "val_Y = Y[last_train_index:last_val_index]\n",
    "test_Y = Y[first_test_index :]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "67095"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(train_X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(67095, 6)"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_Y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from keras.preprocessing.text import Tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "my_tokenizer = Tokenizer(num_words=10000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "my_tokenizer.fit_on_texts(train_X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "122851"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#our dictionary \n",
    "len(set(list(my_tokenizer.word_index.values())))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "train_X_tokenized = my_tokenizer.texts_to_sequences(train_X)\n",
    "val_X_tokenized = my_tokenizer.texts_to_sequences(val_X)\n",
    "test_X_tokenized = my_tokenizer.texts_to_sequences(test_X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"Nonsense?  kiss off, geek. what I said is true.  I'll have your account terminated.\""
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_X[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[799, 4690, 212, 6731, 40, 7, 166, 8, 398, 235, 19, 20, 355]"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_X_tokenized[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# task : define a keras model for this task, look at the rnn example as reference"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from keras.models import Sequential\n",
    "from keras.layers.embeddings import Embedding\n",
    "from keras.preprocessing import sequence\n",
    "from keras.layers.recurrent import LSTM\n",
    "from keras.layers import Dense, Activation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "X_train_tokenized_padded = sequence.pad_sequences(train_X_tokenized, maxlen=20)\n",
    "X_val_tokenized_padded = sequence.pad_sequences(val_X_tokenized, maxlen=20)\n",
    "X_test_tokenized_padded = sequence.pad_sequences(test_X_tokenized, maxlen=20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[799, 4690, 212, 6731, 40, 7, 166, 8, 398, 235, 19, 20, 355]"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_X_tokenized[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([   0,    0,    0,    0,    0,    0,    0,  799, 4690,  212, 6731,\n",
       "         40,    7,  166,    8,  398,  235,   19,   20,  355], dtype=int32)"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train_tokenized_padded[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/1\n",
      "67095/67095 [==============================] - 250s 4ms/step - loss: 0.2996 - acc: 0.9931\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7f476be0ac88>"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = Sequential()\n",
    "model.add(Embedding(input_dim = 10000, output_dim = 128 ))\n",
    "model.add(LSTM(300))\n",
    "model.add(Dense(6))\n",
    "model.add(Activation('sigmoid'))\n",
    "model.compile(loss='categorical_crossentropy', optimizer='adam',metrics=[\"accuracy\"])\n",
    "model.fit(X_train_tokenized_padded,train_Y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# task : write code to feed the model with new sentences "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test_sentence = [\"I will kill you \"] # needs to be an array"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "test_sentence_tokenized = my_tokenizer.texts_to_sequences(test_sentence)\n",
    "test_sentence_tokenized_padded = sequence.pad_sequences (test_sentence_tokenized, maxlen = 20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "predictions = model.predict(test_sentence_tokenized_padded)\n",
    "class_list # 1, 0, 1, 0, 1, 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "for i in range(len(class_list)):\n",
    "    print (class_list[i] + \" prediction is \" + str(predictions[0][i]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "assert 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "plt.plot(val_Y_pred[5],'bo')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "obscene_pred =  val_Y_pred[:,2]\n",
    "obscene_gt = val_Y[:,2]\n",
    "gt_vec = np.where(obscene_gt ==1)[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "pred_vec = np.where(obscene_pred>0.7)[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "sentences_indices_on_which_we_were_correct = set(pred_vec).intersection(set(gt_vec))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "recall = 100 *len(sentences_indices_on_which_we_were_correct)/ float(len(gt_vec)) \n",
    "precision  = 100 * (len(sentences_indices_on_which_we_were_correct) / float(len(pred_vec)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "print ( 'our recall is {}'.format (recall))\n",
    "print ('but we have {} precision'.format(precision))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import precision_recall_curve"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "precision, recall, thresholds = precision_recall_curve(obscene_gt,obscene_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "thresholds[::1000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "recall[::100]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "precision[::1000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "plt.plot(recall[::1000],precision[::1000])\n",
    "plt.xlabel('recall')\n",
    "plt.ylabel('precision')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}