AmazonMusicReviewsSentimentAnalysisWithPredictiom.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(112197, 30)\n"
     ]
    }
   ],
   "source": [
    "data = pd.read_csv(\"cleanresult.csv\", header=0, delimiter=\",\")\n",
    "\n",
    "data['lsentiment']=0\n",
    "print(data.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Taking sentiment of data using Sentiwordnet method"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#WordNet constructs are lemmas and synset\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "from nltk.corpus import wordnet as wn\n",
    "#SentiWordNet is a lexical resource for opinion mining.\n",
    "#SentiWordNet assigns to each synset of WordNet three\n",
    "#sentiment scores: positivity, negativity, and objectivity\n",
    "from nltk.corpus import sentiwordnet as swn\n",
    "from nltk import sent_tokenize, word_tokenize, pos_tag"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "lemmatizer = WordNetLemmatizer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# method for part of speech (POS) tagging\n",
    "def penn_to_wn(tag):\n",
    "     if tag.startswith('J'):\n",
    "        return wn.ADJ\n",
    "     elif tag.startswith('N'):\n",
    "        return wn.NOUN\n",
    "     elif tag.startswith('R'):\n",
    "        return wn.ADV\n",
    "     elif tag.startswith('V'):\n",
    "        return wn.VERB\n",
    "     return None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#method to find the sentiment.\n",
    "#1. first word is tokenized\n",
    "#2. checked if the tokenized word is part of speech\n",
    "#3. checked if a word belongs to lemma (root form) using lemmatizer\n",
    "#4. checked if a word is making some meaning using sysnset and pos tagging\n",
    "#if all the above criteria is met, we take sentiment of the word from synset using sentiWordNet\n",
    "def swn_polarity(text):\n",
    "    sentiment = 0\n",
    "    tokens_count = 0\n",
    "    #tokenizing\n",
    "    for raw_sentence in text:\n",
    "        tagged_sentence = pos_tag(word_tokenize(raw_sentence))\n",
    "        #check if word is in pos\n",
    "        for word, tag in tagged_sentence:\n",
    "            wn_tag = penn_to_wn(tag)\n",
    "            if wn_tag not in (wn.NOUN, wn.ADJ, wn.ADV):\n",
    "                continue\n",
    "            #Check if its  a lemma\n",
    "            lemma = lemmatizer.lemmatize(word, pos=wn_tag)\n",
    "            if not lemma:\n",
    "                continue\n",
    "            #check if its in synsets\n",
    "            synsets = wn.synsets(lemma, pos=wn_tag)\n",
    "            if not synsets:\n",
    "                continue\n",
    " \n",
    "            \n",
    "            synset = synsets[0]\n",
    "            swn_synset = swn.senti_synset(synset.name())\n",
    "            #sentiment calculating\n",
    "            sentiment += swn_synset.pos_score() - swn_synset.neg_score()\n",
    "            tokens_count += 1\n",
    "            \n",
    "            if not tokens_count:\n",
    "                return 0\n",
    "            if sentiment >= 0:\n",
    "                return 1\n",
    "            \n",
    "\n",
    "            return 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#taking users review data for analysis\n",
    "textdata1 = data[\"reviewText\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#calling method to calc sentimemt\n",
    "lsentiment=[]\n",
    "\n",
    "for i in textdata1:\n",
    "    lsentiment.append(swn_polarity(i))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "resultsentiment=zip(textdata1,lsentiment)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "done\n"
     ]
    }
   ],
   "source": [
    "#adding sentiment result to dataset\n",
    "data['lsentiment']=pd.Series(lsentiment)\n",
    "#output data to csv for analysis\n",
    "#review.to_csv('output_SentimentResult.csv')\n",
    "print \"done\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#Prediction : Creating model for sentiment analysis using confidence interval values in dataset\n",
    "\n",
    "#Run a logistic regression\n",
    "import statsmodels.api as sm2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#creating training set and test set and removing nan values\n",
    "train = data[:90000] \n",
    "test = data[90000:] \n",
    "train = train.fillna(0)\n",
    "test=test.fillna(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Optimization terminated successfully.\n",
      "         Current function value: 0.306467\n",
      "         Iterations 6\n",
      "                           Logit Regression Results                           \n",
      "==============================================================================\n",
      "Dep. Variable:             lsentiment   No. Observations:                90000\n",
      "Model:                          Logit   Df Residuals:                    89999\n",
      "Method:                           MLE   Df Model:                            0\n",
      "Date:                Mon, 27 Feb 2017   Pseudo R-squ.:                -0.05248\n",
      "Time:                        15:51:59   Log-Likelihood:                -27582.\n",
      "converged:                       True   LL-Null:                       -26207.\n",
      "                                        LLR p-value:                       nan\n",
      "==============================================================================\n",
      "                 coef    std err          z      P>|z|      [95.0% Conf. Int.]\n",
      "------------------------------------------------------------------------------\n",
      "overall        0.5217      0.003    191.052      0.000         0.516     0.527\n",
      "==============================================================================\n"
     ]
    }
   ],
   "source": [
    "#make prediction model using overall rating as independent variable ,predict sentiment\n",
    "\n",
    "# Run a logistic regression \n",
    "import statsmodels.api as sm2\n",
    "logit=sm2.Logit(train['lsentiment'].dropna(),train['overall'].dropna())\n",
    "result=logit.fit() \n",
    "print(result.summary())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1 1 1 ..., 1 1 1]\n"
     ]
    }
   ],
   "source": [
    "#prediction of sentiment based on confidence\n",
    "predictSentiment = result.predict(test['overall'])\n",
    "\n",
    "predictSentiment = (predictSentiment > 0.50).astype(int)\n",
    "print(predictSentiment)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Confusion Matrix\n",
      "[[    0  1860]\n",
      " [    0 20337]]\n"
     ]
    }
   ],
   "source": [
    "print(\"Confusion Matrix\")\n",
    "from sklearn.metrics import confusion_matrix\n",
    "\n",
    "print(confusion_matrix(test['lsentiment'],predictSentiment))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.916204892553\n"
     ]
    }
   ],
   "source": [
    "#Accuracy of setiment analysis \n",
    "from sklearn.metrics import accuracy_score\n",
    "print(accuracy_score(test['lsentiment'],predictSentiment))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}