antivac-data.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Experiments with the antivac data of Kunneman et al 2020\n",
    "\n",
    "F. Kunneman, M. Lambooij, A. Wong, A. van den Bosch, L. Mollema. Monitoring stance towards vaccination in Twitter messages. In: BMC Medical Informatics and Decision Making. 20, 1, p. 1-14, 2020, doi 10.1186/s12911-020-1046-y. Data: http://cls.ru.nl/~fkunneman/data_stance_vaccination.zip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "import fasttext\n",
    "import json\n",
    "import os\n",
    "import pandas as pd\n",
    "import random\n",
    "import re\n",
    "from nltk.tokenize import TweetTokenizer\n",
    "from IPython.display import clear_output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "DATADIR = \"/home/erikt/projects/puregome/data/data_stance_vaccination/\"\n",
    "DATAFILE = \"tweetids_labels.txt\"\n",
    "TWEETSFILE = \"getTweetsId.py.out\"\n",
    "POLARITY = \"Polarity\"\n",
    "SENTIMENT = \"Sentiment\"\n",
    "IDSTR = \"id_str\"\n",
    "TEXT = \"text\"\n",
    "TOKENIZEDTEXT = \"tokenizedtext\"\n",
    "LABELINGTYPE = \"Labeling type\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "def readAnnotations(inFileName):\n",
    "    return(pd.read_csv(inFileName,sep=\"\\t\",index_col=\"Tweet ID\"))\n",
    "           \n",
    "dfAnnotations = readAnnotations(DATADIR+DATAFILE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Labeling type</th>\n",
       "      <th>Binary</th>\n",
       "      <th>Irrelevance Filter</th>\n",
       "      <th>Polarity</th>\n",
       "      <th>Polarity + Sentiment</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Tweet ID</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>560765984135737346</td>\n",
       "      <td>strict</td>\n",
       "      <td>Other</td>\n",
       "      <td>Irrelevant</td>\n",
       "      <td>Irrelevant</td>\n",
       "      <td>Irrelevant</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>354874289536761856</td>\n",
       "      <td>strict</td>\n",
       "      <td>Other</td>\n",
       "      <td>Other</td>\n",
       "      <td>Positive</td>\n",
       "      <td>Positive + Other</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>353117242700988416</td>\n",
       "      <td>strict</td>\n",
       "      <td>Other</td>\n",
       "      <td>Other</td>\n",
       "      <td>Positive</td>\n",
       "      <td>Positive + Frustration</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>534595037599105024</td>\n",
       "      <td>strict</td>\n",
       "      <td>Other</td>\n",
       "      <td>Irrelevant</td>\n",
       "      <td>Irrelevant</td>\n",
       "      <td>Irrelevant</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>790283805324152832</td>\n",
       "      <td>strict</td>\n",
       "      <td>Other</td>\n",
       "      <td>Other</td>\n",
       "      <td>Positive</td>\n",
       "      <td>Positive + Other</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>790103347877535744</td>\n",
       "      <td>one</td>\n",
       "      <td>Negative</td>\n",
       "      <td>Negative</td>\n",
       "      <td>Negative</td>\n",
       "      <td>Negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>799739264128913408</td>\n",
       "      <td>one</td>\n",
       "      <td>Negative</td>\n",
       "      <td>Negative</td>\n",
       "      <td>Negative</td>\n",
       "      <td>Negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>746765559484776448</td>\n",
       "      <td>one</td>\n",
       "      <td>Negative</td>\n",
       "      <td>Negative</td>\n",
       "      <td>Negative</td>\n",
       "      <td>Negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>599494572859883520</td>\n",
       "      <td>one</td>\n",
       "      <td>Negative</td>\n",
       "      <td>Negative</td>\n",
       "      <td>Negative</td>\n",
       "      <td>Negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>356145576909340674</td>\n",
       "      <td>one</td>\n",
       "      <td>Other</td>\n",
       "      <td>Irrelevant</td>\n",
       "      <td>Irrelevant</td>\n",
       "      <td>Irrelevant</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8268 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                   Labeling type    Binary Irrelevance Filter    Polarity  \\\n",
       "Tweet ID                                                                    \n",
       "560765984135737346        strict     Other         Irrelevant  Irrelevant   \n",
       "354874289536761856        strict     Other              Other    Positive   \n",
       "353117242700988416        strict     Other              Other    Positive   \n",
       "534595037599105024        strict     Other         Irrelevant  Irrelevant   \n",
       "790283805324152832        strict     Other              Other    Positive   \n",
       "...                          ...       ...                ...         ...   \n",
       "790103347877535744           one  Negative           Negative    Negative   \n",
       "799739264128913408           one  Negative           Negative    Negative   \n",
       "746765559484776448           one  Negative           Negative    Negative   \n",
       "599494572859883520           one  Negative           Negative    Negative   \n",
       "356145576909340674           one     Other         Irrelevant  Irrelevant   \n",
       "\n",
       "                      Polarity + Sentiment  \n",
       "Tweet ID                                    \n",
       "560765984135737346              Irrelevant  \n",
       "354874289536761856        Positive + Other  \n",
       "353117242700988416  Positive + Frustration  \n",
       "534595037599105024              Irrelevant  \n",
       "790283805324152832        Positive + Other  \n",
       "...                                    ...  \n",
       "790103347877535744                Negative  \n",
       "799739264128913408                Negative  \n",
       "746765559484776448                Negative  \n",
       "599494572859883520                Negative  \n",
       "356145576909340674              Irrelevant  \n",
       "\n",
       "[8268 rows x 5 columns]"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dfAnnotations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "TEXT = \"text\"\n",
    "FULLTEXT = \"full_text\"\n",
    "EXTENDEDTWEET = \"extended_tweet\"\n",
    "RETWEETEDSTATUS = \"retweeted_status\"\n",
    "\n",
    "def getTweetText(jsonData):\n",
    "    text = \"\"\n",
    "    if TEXT in jsonData: \n",
    "        text = jsonData[TEXT]\n",
    "    if EXTENDEDTWEET in jsonData and \\\n",
    "       FULLTEXT in jsonData[EXTENDEDTWEET]:\n",
    "        text = jsonData[EXTENDEDTWEET][FULLTEXT]\n",
    "    if RETWEETEDSTATUS in jsonData and \\\n",
    "       EXTENDEDTWEET in jsonData[RETWEETEDSTATUS] and \\\n",
    "       FULLTEXT in jsonData[RETWEETEDSTATUS][EXTENDEDTWEET]:\n",
    "        text = jsonData[RETWEETEDSTATUS][EXTENDEDTWEET][FULLTEXT]\n",
    "    return(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "def readTweetTexts(inFileName):\n",
    "    tweetTexts = {}\n",
    "    inFile = open(inFileName,\"r\")\n",
    "    for line in inFile:\n",
    "        jsonData = json.loads(line.strip())\n",
    "        idStr = jsonData[IDSTR]\n",
    "        text = getTweetText(jsonData)\n",
    "        tweetTexts[idStr] = text\n",
    "    inFile.close()\n",
    "    return(tweetTexts)\n",
    "\n",
    "dictTweetTexts = readTweetTexts(DATADIR+TWEETSFILE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "def makeFasttextData(dfAnnotations,dictTweetTexts,labelingType=\"\"):\n",
    "    fasttextData = []\n",
    "    for idStr in dictTweetTexts:\n",
    "        polarity = re.sub(\" \",\"_\",dfAnnotations.loc[int(idStr)][POLARITY])\n",
    "        tokenizedText = \" \".join(TweetTokenizer().tokenize(dictTweetTexts[idStr])).lower()\n",
    "        if labelingType == \"\" or dfAnnotations.loc[int(idStr)][LABELINGTYPE] == labelingType:\n",
    "            fasttextData.append({IDSTR:idStr,POLARITY:polarity,TOKENIZEDTEXT:tokenizedText})\n",
    "    return(fasttextData)\n",
    "        \n",
    "fasttextData = makeFasttextData(dfAnnotations,dictTweetTexts,labelingType=\"strict\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2052"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(fasttextData)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "PRETRAINEDDIR = \"/home/erikt/projects/newsgac/fasttext-runs/\"\n",
    "WIKIFILENAME = \"wiki.nl.vec\"\n",
    "DIM = 300\n",
    "EPOCH = 100\n",
    "LR = 0.1\n",
    "N = 10\n",
    "LARGEINT = 9999999999\n",
    "LABELPREFIX = \"__label__\"\n",
    "TRAIN = \"TRAIN\"+str(int(random.random()*LARGEINT))\n",
    "TEST = \"TEST\"+str(int(random.random()*LARGEINT))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "def runFasttext(fasttextData,dim=DIM,epoch=EPOCH,lr=LR,n=N):\n",
    "    predictionCounts = []\n",
    "    predictionLabels = []\n",
    "    for fold in range(0,n):\n",
    "        clear_output(wait=True)\n",
    "        print(\"starting fold\",fold)\n",
    "        testStart = round(fold*len(fasttextData)/n)\n",
    "        testEnd = round((fold+1)*len(fasttextData)/n)\n",
    "        trainFile = open(TRAIN,\"w\")\n",
    "        testFile = open(TEST,\"w\")\n",
    "        testData = []\n",
    "        for i in range(0,len(fasttextData)):\n",
    "            data = LABELPREFIX+fasttextData[i][POLARITY]+\" \"+fasttextData[i][TOKENIZEDTEXT]\n",
    "            if i < testStart or i >= testEnd: \n",
    "                print(data,file=trainFile)\n",
    "            else: \n",
    "                print(data,file=testFile)\n",
    "                testData.append(data)\n",
    "        testFile.close()\n",
    "        trainFile.close()\n",
    "        model = fasttext.train_supervised(TRAIN,dim=dim,epoch=epoch,lr=lr) #,pretrainedVectors=PRETRAINEDDIR+WIKIFILENAME)\n",
    "        predictionCounts.append([*model.test(TEST)])\n",
    "        predictionLabels.extend(model.predict(testData)[0])\n",
    "        os.unlink(TRAIN)\n",
    "        os.unlink(TEST)\n",
    "    clear_output(wait=True)\n",
    "    print(\"finished\")\n",
    "    return(predictionCounts,predictionLabels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "finished\n"
     ]
    }
   ],
   "source": [
    "predictionCounts,predictionLabels = runFasttext(fasttextData,dim=300,epoch=50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cases: 2052; precision: 0.533; recall: 0.533\n"
     ]
    }
   ],
   "source": [
    "def showOverallPrecision(predictionCounts):\n",
    "    caseTotal = 0\n",
    "    pTotal = 0\n",
    "    rTotal = 0\n",
    "    for i in range(0,len(predictionCounts)):\n",
    "        caseTotal += predictionCounts[i][0]\n",
    "        pTotal += predictionCounts[i][0]*predictionCounts[i][1]\n",
    "        rTotal += predictionCounts[i][0]*predictionCounts[i][2]\n",
    "    print(\"cases: {0}; precision: {1}; recall: {2}\".format(caseTotal,round(pTotal/caseTotal,3),round(rTotal/caseTotal,3)))\n",
    "    \n",
    "showOverallPrecision(predictionCounts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "precision: 0.344 recall: 0.276 f1: 0.307 Negative\n",
      "precision: 0.234 recall: 0.15 f1: 0.182 Not_clear\n",
      "precision: 0.313 recall: 0.268 f1: 0.289 Neutral\n",
      "precision: 0.496 recall: 0.504 f1: 0.5 Irrelevant\n",
      "precision: 0.649 recall: 0.739 f1: 0.691 Positive\n"
     ]
    }
   ],
   "source": [
    "GOLD = \"gold\"\n",
    "PREDICTED = \"predicted\"\n",
    "CORRECT = \"correct\"\n",
    "\n",
    "labelResults = {}\n",
    "for i in range(0,len(fasttextData)):\n",
    "    labelGold = fasttextData[i][POLARITY]\n",
    "    labelPredicted = re.sub(LABELPREFIX,\"\",predictionLabels[i][0])\n",
    "    if not labelGold in labelResults: labelResults[labelGold] = {GOLD:0,PREDICTED:0,CORRECT:0}\n",
    "    if not labelPredicted in labelResults: labelResults[labelPredicted] = {GOLD:0,PREDICTED:0,CORRECT:0}\n",
    "    labelResults[labelGold][GOLD] += 1\n",
    "    labelResults[labelPredicted][PREDICTED] += 1\n",
    "    if labelGold == labelPredicted: labelResults[labelGold][CORRECT] += 1\n",
    "for label in labelResults:\n",
    "    precision = labelResults[label][CORRECT]/labelResults[label][PREDICTED]\n",
    "    recall = labelResults[label][CORRECT]/labelResults[label][GOLD]\n",
    "    f1 = 2*precision*recall/(precision+recall)\n",
    "    print(\"precision:\",round(precision,3),\"recall:\",round(recall,3),\"f1:\",round(f1,3),label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Negative': 228,\n",
       " 'Not_clear': 167,\n",
       " 'Irrelevant': 421,\n",
       " 'Positive': 982,\n",
       " 'Neutral': 254}"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def countGoldLabels(fasttextData):\n",
    "    labelCountsGold = {}\n",
    "    for i in range(0,len(fasttextData)):\n",
    "        label = fasttextData[i][POLARITY]\n",
    "        if label in labelCountsGold: labelCountsGold[label] += 1\n",
    "        else: labelCountsGold[label] = 1\n",
    "    return(labelCountsGold)\n",
    "\n",
    "labelCountsGold = countGoldLabels(fasttextData)\n",
    "labelCountsGold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "baseline accuracy: 0.479\n"
     ]
    }
   ],
   "source": [
    "def getBaselineAccuracy(labelCountsGold):\n",
    "    return(round(max(labelCountsGold.values())/sum(labelCountsGold.values()),3))\n",
    "\n",
    "print(\"baseline accuracy:\",getBaselineAccuracy(labelCountsGold))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'__label__Negative': 183,\n",
       " '__label__Neutral': 217,\n",
       " '__label__Irrelevant': 427,\n",
       " '__label__Positive': 1118,\n",
       " '__label__Not_clear': 107}"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def countPredictedLabels(predictionLabels):\n",
    "    labelCountsPredicted = {}\n",
    "    for i in range(0,len(predictionLabels)):\n",
    "        label = predictionLabels[i][0]\n",
    "        if label in labelCountsPredicted: labelCountsPredicted[label] += 1\n",
    "        else: labelCountsPredicted[label] = 1\n",
    "    return(labelCountsPredicted)\n",
    "\n",
    "labelCountsPredicted = countPredictedLabels(predictionLabels)\n",
    "labelCountsPredicted"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>col_0</th>\n",
       "      <th>__label__Irrelevant</th>\n",
       "      <th>__label__Negative</th>\n",
       "      <th>__label__Neutral</th>\n",
       "      <th>__label__Not_clear</th>\n",
       "      <th>__label__Positive</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>row_0</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>Irrelevant</td>\n",
       "      <td>212</td>\n",
       "      <td>27</td>\n",
       "      <td>38</td>\n",
       "      <td>22</td>\n",
       "      <td>122</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>Negative</td>\n",
       "      <td>37</td>\n",
       "      <td>63</td>\n",
       "      <td>20</td>\n",
       "      <td>15</td>\n",
       "      <td>93</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>Neutral</td>\n",
       "      <td>41</td>\n",
       "      <td>19</td>\n",
       "      <td>68</td>\n",
       "      <td>17</td>\n",
       "      <td>109</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>Not_clear</td>\n",
       "      <td>33</td>\n",
       "      <td>18</td>\n",
       "      <td>23</td>\n",
       "      <td>25</td>\n",
       "      <td>68</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>Positive</td>\n",
       "      <td>104</td>\n",
       "      <td>56</td>\n",
       "      <td>68</td>\n",
       "      <td>28</td>\n",
       "      <td>726</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "col_0       __label__Irrelevant  __label__Negative  __label__Neutral  \\\n",
       "row_0                                                                  \n",
       "Irrelevant                  212                 27                38   \n",
       "Negative                     37                 63                20   \n",
       "Neutral                      41                 19                68   \n",
       "Not_clear                    33                 18                23   \n",
       "Positive                    104                 56                68   \n",
       "\n",
       "col_0       __label__Not_clear  __label__Positive  \n",
       "row_0                                              \n",
       "Irrelevant                  22                122  \n",
       "Negative                    15                 93  \n",
       "Neutral                     17                109  \n",
       "Not_clear                   25                 68  \n",
       "Positive                    28                726  "
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def makeConfusionMatrix(fasttextData,predictionLabels):\n",
    "    goldLabels = pd.Series([fasttextData[i][POLARITY] for i in range(0,len(fasttextData))])\n",
    "    predictedLabels = pd.Series([predictionLabels[i][0] for i in range(0,len(predictionLabels))])\n",
    "    return(pd.crosstab(goldLabels,predictedLabels))\n",
    "\n",
    "makeConfusionMatrix(fasttextData,predictionLabels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total absolute deviation 284\n"
     ]
    }
   ],
   "source": [
    "print(\"total absolute deviation\",sum([abs(labelCountsPredicted[LABELPREFIX+l]-labelCountsGold[l]) for l in labelCountsGold]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "relative deviation per label: {'Negative': -0.197, 'Not_clear': -0.359, 'Irrelevant': 0.014, 'Positive': 0.138, 'Neutral': -0.146}\n"
     ]
    }
   ],
   "source": [
    "print(\"relative deviation per label:\",\\\n",
    "      {l:round((labelCountsPredicted[LABELPREFIX+l]-labelCountsGold[l])/labelCountsGold[l],3) for l in labelCountsGold})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Combine stance data sets with different topics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [],
   "source": [
    "TOPIC = \"mondkapje\"\n",
    "FILETWEETS = TOPIC+\"-tweets.csv\"\n",
    "FILEANNOTATIONS = \"human-labels-\"+TOPIC+\"-tweets.txt\"\n",
    "LABEL = \"label\"\n",
    "TEXT = \"text\"\n",
    "USER = \"user\"\n",
    "DATASIZEANTIVAC = len(fasttextData)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [],
   "source": [
    "def readAnnotations(inFileName):\n",
    "    return(pd.read_csv(inFileName,header=None,sep=\" \"))\n",
    "\n",
    "dfAnnotations = readAnnotations(FILEANNOTATIONS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [],
   "source": [
    "def readTweets(inFileName):\n",
    "    return(pd.read_csv(inFileName,header=None,index_col=0))\n",
    "\n",
    "dfTweets = readTweets(FILETWEETS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": [
    "def convertLabel(label):\n",
    "    label = re.sub(\"NEUTRAL\",\"Neutral\",label)\n",
    "    label = re.sub(\"POSITIVE\",\"Positive\",label)\n",
    "    label = re.sub(\"NEGATIVE\",\"Negative\",label)\n",
    "    label = re.sub(\"IRRELEVANT\",\"Irrelevant\",label)\n",
    "    return(label)\n",
    "\n",
    "def makeFasttextData(dfAnnotations,dfTweets):\n",
    "    fasttextData = []\n",
    "    seen = {}\n",
    "    for i in range(0,len(dfAnnotations)):\n",
    "        tweetId = dfAnnotations.iloc[i][2]\n",
    "        if tweetId in list(dfTweets.index):\n",
    "            tweetLabel = convertLabel(dfAnnotations.iloc[i][4])\n",
    "            tweetUser = dfTweets.loc[tweetId][1]\n",
    "            tokenizedText = \" \".join(TweetTokenizer().tokenize(dfTweets.loc[tweetId][2])).lower()\n",
    "            if not tokenizedText in seen:\n",
    "                seen[tokenizedText] = True\n",
    "                fasttextData.append({IDSTR:tweetId,POLARITY:tweetLabel,TOKENIZEDTEXT:tokenizedText})\n",
    "    return(fasttextData)\n",
    "\n",
    "fasttextDataTopic = makeFasttextData(dfAnnotations,dfTweets)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    "fasttextDataCombined = fasttextData+fasttextDataTopic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2052, 593, 2645)"
      ]
     },
     "execution_count": 96,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(fasttextData),len(fasttextDataTopic),len(fasttextDataCombined)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "finished\n"
     ]
    }
   ],
   "source": [
    "predictionCounts,predictionLabels = runFasttext(fasttextDataCombined,dim=300,epoch=50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cases: 2645; precision: 0.544; recall: 0.544\n"
     ]
    }
   ],
   "source": [
    "showOverallPrecision(predictionCounts)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Results data section antivac"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Negative': 228,\n",
       " 'Not_clear': 167,\n",
       " 'Irrelevant': 421,\n",
       " 'Positive': 982,\n",
       " 'Neutral': 254}"
      ]
     },
     "execution_count": 101,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "labelCountsGold = countGoldLabels(fasttextDataCombined[:DATASIZEANTIVAC])\n",
    "labelCountsGold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "baseline accuracy: 0.479\n"
     ]
    }
   ],
   "source": [
    "print(\"baseline accuracy:\",getBaselineAccuracy(labelCountsGold))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'__label__Negative': 184,\n",
       " '__label__Neutral': 218,\n",
       " '__label__Irrelevant': 441,\n",
       " '__label__Positive': 1112,\n",
       " '__label__Not_clear': 97}"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "labelCountsPredicted = countPredictedLabels(predictionLabels[:DATASIZEANTIVAC])\n",
    "labelCountsPredicted"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>col_0</th>\n",
       "      <th>__label__Irrelevant</th>\n",
       "      <th>__label__Negative</th>\n",
       "      <th>__label__Neutral</th>\n",
       "      <th>__label__Not_clear</th>\n",
       "      <th>__label__Positive</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>row_0</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>Irrelevant</td>\n",
       "      <td>214</td>\n",
       "      <td>19</td>\n",
       "      <td>39</td>\n",
       "      <td>21</td>\n",
       "      <td>128</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>Negative</td>\n",
       "      <td>41</td>\n",
       "      <td>68</td>\n",
       "      <td>19</td>\n",
       "      <td>11</td>\n",
       "      <td>89</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>Neutral</td>\n",
       "      <td>47</td>\n",
       "      <td>13</td>\n",
       "      <td>65</td>\n",
       "      <td>24</td>\n",
       "      <td>105</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>Not_clear</td>\n",
       "      <td>30</td>\n",
       "      <td>22</td>\n",
       "      <td>28</td>\n",
       "      <td>19</td>\n",
       "      <td>68</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>Positive</td>\n",
       "      <td>109</td>\n",
       "      <td>62</td>\n",
       "      <td>67</td>\n",
       "      <td>22</td>\n",
       "      <td>722</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "col_0       __label__Irrelevant  __label__Negative  __label__Neutral  \\\n",
       "row_0                                                                  \n",
       "Irrelevant                  214                 19                39   \n",
       "Negative                     41                 68                19   \n",
       "Neutral                      47                 13                65   \n",
       "Not_clear                    30                 22                28   \n",
       "Positive                    109                 62                67   \n",
       "\n",
       "col_0       __label__Not_clear  __label__Positive  \n",
       "row_0                                              \n",
       "Irrelevant                  21                128  \n",
       "Negative                    11                 89  \n",
       "Neutral                     24                105  \n",
       "Not_clear                   19                 68  \n",
       "Positive                    22                722  "
      ]
     },
     "execution_count": 104,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "makeConfusionMatrix(fasttextDataCombined[:DATASIZEANTIVAC],predictionLabels[:DATASIZEANTIVAC])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total absolute deviation 300\n"
     ]
    }
   ],
   "source": [
    "print(\"total absolute deviation\",sum([abs(labelCountsPredicted[LABELPREFIX+l]-labelCountsGold[l]) for l in labelCountsGold]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "relative deviation per label: {'Negative': -0.193, 'Not_clear': -0.419, 'Irrelevant': 0.048, 'Positive': 0.132, 'Neutral': -0.142}\n"
     ]
    }
   ],
   "source": [
    "print(\"relative deviation per label:\",\\\n",
    "      {l:round((labelCountsPredicted[LABELPREFIX+l]-labelCountsGold[l])/labelCountsGold[l],3) for l in labelCountsGold})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Results data section mondkapje"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Neutral': 53, 'Irrelevant': 236, 'Negative': 272, 'Positive': 32}"
      ]
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "labelCountsGold = countGoldLabels(fasttextDataCombined[DATASIZEANTIVAC:])\n",
    "labelCountsGold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "baseline accuracy: 0.459\n"
     ]
    }
   ],
   "source": [
    "print(\"baseline accuracy:\",getBaselineAccuracy(labelCountsGold))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'__label__Neutral': 30,\n",
       " '__label__Irrelevant': 225,\n",
       " '__label__Negative': 315,\n",
       " '__label__Positive': 19,\n",
       " '__label__Not_clear': 4}"
      ]
     },
     "execution_count": 110,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "labelCountsPredicted = countPredictedLabels(predictionLabels[DATASIZEANTIVAC:])\n",
    "labelCountsPredicted"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>col_0</th>\n",
       "      <th>__label__Irrelevant</th>\n",
       "      <th>__label__Negative</th>\n",
       "      <th>__label__Neutral</th>\n",
       "      <th>__label__Not_clear</th>\n",
       "      <th>__label__Positive</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>row_0</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>Irrelevant</td>\n",
       "      <td>147</td>\n",
       "      <td>72</td>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>Negative</td>\n",
       "      <td>51</td>\n",
       "      <td>195</td>\n",
       "      <td>11</td>\n",
       "      <td>3</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>Neutral</td>\n",
       "      <td>16</td>\n",
       "      <td>29</td>\n",
       "      <td>8</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>Positive</td>\n",
       "      <td>11</td>\n",
       "      <td>19</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "col_0       __label__Irrelevant  __label__Negative  __label__Neutral  \\\n",
       "row_0                                                                  \n",
       "Irrelevant                  147                 72                10   \n",
       "Negative                     51                195                11   \n",
       "Neutral                      16                 29                 8   \n",
       "Positive                     11                 19                 1   \n",
       "\n",
       "col_0       __label__Not_clear  __label__Positive  \n",
       "row_0                                              \n",
       "Irrelevant                   1                  6  \n",
       "Negative                     3                 12  \n",
       "Neutral                      0                  0  \n",
       "Positive                     0                  1  "
      ]
     },
     "execution_count": 111,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "makeConfusionMatrix(fasttextDataCombined[DATASIZEANTIVAC:],predictionLabels[DATASIZEANTIVAC:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total absolute deviation 90\n"
     ]
    }
   ],
   "source": [
    "print(\"total absolute deviation\",sum([abs(labelCountsPredicted[LABELPREFIX+l]-labelCountsGold[l]) for l in labelCountsGold]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "relative deviation per label: {'Neutral': -0.434, 'Irrelevant': -0.047, 'Negative': 0.158, 'Positive': -0.406}\n"
     ]
    }
   ],
   "source": [
    "print(\"relative deviation per label:\",\\\n",
    "      {l:round((labelCountsPredicted[LABELPREFIX+l]-labelCountsGold[l])/labelCountsGold[l],3) for l in labelCountsGold})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The results of the data section mondkapje remain bad. This can be explained by two facts: 1. the distribution whithin the two data sections is quite different (antivac: mostly positive; mondkapje: mostly negative) and 2. the antivac data is much larger than the mondkapje data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}