sentiment-pattern.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Sentiment\n",
    "\n",
    "Visualize sentiment analysis. This notebook relies on sentiment scores per tweet stored in the directory `data/sentiment/pattern` . These sentiment scores can be generated with the script `scripts/sentiment-pattern-text.py` ."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import datetime\n",
    "import os\n",
    "import re\n",
    "import statistics\n",
    "import sys\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.dates as mdates\n",
    "from IPython.display import clear_output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pandas.plotting import register_matplotlib_converters\n",
    "register_matplotlib_converters()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "DATASOURCE = \"twitter\"\n",
    "DATADIRECTORYALL = f\"../data/{DATASOURCE}/sentiment/\"\n",
    "DATADIRECTORYTEXT = f\"../data/{DATASOURCE}/text/\"\n",
    "SENTIMENT = \"sentiment\"\n",
    "COUNT = \"count\"\n",
    "DATA = \"data\"\n",
    "LABEL = \"label\"\n",
    "HIGHLIGHT = \"highlight\"\n",
    "HIGHLIGHTLABEL = \"highlightlabel\"\n",
    "TEXT =\"text\"\n",
    "IDSTR = \"id_str\"\n",
    "DATE = \"date\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def squeal(text=None):\n",
    "    clear_output(wait=True)\n",
    "    if not text is None: print(text)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Read hourly sentiment analysis by pattern's sentiment analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "DEFAULTFILEPATTERN = \"2.*z\"\n",
    "\n",
    "def getSentimentPerHour(dataDirectory,filePattern=DEFAULTFILEPATTERN):\n",
    "    fileList = sorted(os.listdir(dataDirectory))\n",
    "    sentimentPerHour = {}\n",
    "    for inFileName in fileList:\n",
    "        if re.search(filePattern,inFileName):\n",
    "            if re.search(r\"-00.out.gz\",inFileName): squeal(inFileName)\n",
    "            try: df = pd.read_csv(dataDirectory+inFileName,compression=\"gzip\",header=None)\n",
    "            except: continue\n",
    "            sentiment = sum(df[1])/len(df)\n",
    "            hour = inFileName[0:11]\n",
    "            sentimentPerHour[hour] = { SENTIMENT:sentiment, COUNT:len(df) }\n",
    "    sentimentPerHour = {key:sentimentPerHour[key] for key in sorted(sentimentPerHour.keys())}\n",
    "    return(sentimentPerHour)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Convert hourly analysis to day scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def makeSentimentPerDay(sentimentPerHour):\n",
    "    sentimentPerDay = {}\n",
    "    for hour in sentimentPerHour:\n",
    "        day = re.sub(\"..$\",\"12\",hour)\n",
    "        if not day in sentimentPerDay: sentimentPerDay[day] = {SENTIMENT:0,COUNT:0}\n",
    "        sentimentPerDay[day][SENTIMENT] += sentimentPerHour[hour][SENTIMENT]*sentimentPerHour[hour][COUNT]\n",
    "        sentimentPerDay[day][COUNT] += sentimentPerHour[hour][COUNT]\n",
    "    for day in sentimentPerDay:\n",
    "        sentimentPerDay[day][SENTIMENT] /= sentimentPerDay[day][COUNT]\n",
    "    return(sentimentPerDay)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Visualize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "DATEFORMATHOUR = \"%Y%m%d-%H\"\n",
    "DATEFORMATMONTH = \"%-d/%-m\"\n",
    "DATEFORMATHRSMINS = \"%H:%M\"\n",
    "DEFAULTTITLE = \"Polarity scores of Dutch tweets over time\"\n",
    "PLOTFILENAME = \"sentiment-all.png\"\n",
    "ANNOTATE = \"annotate\"\n",
    "\n",
    "def stringToDate(dateString):\n",
    "    return(datetime.datetime.strptime(dateString,DATEFORMATHOUR))\n",
    "\n",
    "def list_skip(list,skip_factor=0):\n",
    "    return([list[i] for i in range(0,len(list)) if (i)%(skip_factor+1) == 0])\n",
    "\n",
    "def visualizeSentiment(dataSources,title=DEFAULTTITLE,dateFormat=DATEFORMATMONTH,skip_factor=0,outFileName=PLOTFILENAME, tuple_data_list=[]):\n",
    "    font = {\"size\":16}\n",
    "    matplotlib.rc(\"font\",**font)\n",
    "    #fig,ax = plt.subplots(figsize=(12,6))\n",
    "    fig,ax = plt.subplots(figsize=(16, 9))\n",
    "    #plt.ylim(-0.14,0.16)\n",
    "    ax.xaxis.set_major_formatter(mdates.DateFormatter(dateFormat))\n",
    "    for i in range(0,len(dataSources)):\n",
    "        data = dataSources[i][DATA]\n",
    "        label = dataSources[i][LABEL]\n",
    "        lineData = ax.plot_date(list_skip([stringToDate(key) for key in data if data[key][COUNT] > 1],skip_factor=skip_factor),\n",
    "                                list_skip([data[key][SENTIMENT] for key in data if data[key][COUNT] > 1],skip_factor=skip_factor),\n",
    "                                xdate=True,fmt=\"-\",label=label)\n",
    "        if HIGHLIGHT in dataSources[i]:\n",
    "            highlight = dataSources[i][HIGHLIGHT]\n",
    "            color = lineData[-1].get_color()\n",
    "            if not HIGHLIGHTLABEL in dataSources[i]:\n",
    "                ax.plot_date([stringToDate(key) for key in highlight],\n",
    "                             [data[key][SENTIMENT] for key in highlight],\\\n",
    "                             fmt=\"o\",color=color)\n",
    "            else:\n",
    "                highlightlabel = dataSources[i][HIGHLIGHTLABEL]\n",
    "                ax.plot_date([stringToDate(key) for key in highlight],\n",
    "                             [data[key][SENTIMENT] for key in highlight],\\\n",
    "                             fmt=\"o\",color=color,label=highlightlabel)\n",
    "        if ANNOTATE in dataSources[i]:\n",
    "            for date1,date2,text in dataSources[i][ANNOTATE]:\n",
    "                plt.annotate(text,(stringToDate(date2),data[date1][SENTIMENT]),color=color)\n",
    "    plt.title(title)\n",
    "    plt.legend(framealpha=0.2, bbox_to_anchor=(0,1), loc=\"upper left\")\n",
    "    # plt.legend(framealpha=0.2)\n",
    "    for tuple_data in tuple_data_list:\n",
    "        plt.text(datetime.datetime.strptime(tuple_data[0], DATEFORMATHOUR), tuple_data[1], tuple_data[2], fontsize=12)\n",
    "    plt.savefig(outFileName)\n",
    "    plt.show()\n",
    "    return(ax)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Process data obtained from all Dutch tweets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "press_conferences = [\"20200301-12\", \"20200309-12\", \"20200312-12\", \"20200315-12\", \"20200317-12\", \"20200319-12\", \"20200323-12\", \"20200331-12\",\n",
    "                     \"20200407-12\", \"20200415-12\", \"20200421-12\", \"20200429-12\",\n",
    "                     \"20200506-12\", \"20200513-12\", \"20200519-12\", \"20200527-12\",\\\n",
    "                     \"20200603-12\", \"20200624-12\",\n",
    "                     \"20200722-12\",\n",
    "                     \"20200806-12\", \"20200818-12\",\n",
    "                     \"20200901-12\", \"20200918-12\", \"20200928-12\",\n",
    "                     \"20201013-12\",\n",
    "                     \"20201103-12\", \"20201117-12\",\n",
    "                     \"20201208-12\", \"20201214-12\",\n",
    "                     \"20210112-12\", \"20210120-12\",\n",
    "                     \"20210202-12\", \"20210223-12\",\n",
    "                     \"20210308-12\", \"20210323-12\",\n",
    "                    ]\n",
    "highlight = list(press_conferences)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "annotate = [(\"20200220-12\",\"20200221-12\",\"(1)\"),\\\n",
    "            (\"20200302-12\",\"20200303-12\",\"(2)\"),\\\n",
    "            (\"20200312-12\",\"20200313-12\",\"(3)\"),\\\n",
    "            (\"20200322-12\",\"20200323-12\",\"(4)\"),\\\n",
    "            (\"20200330-12\",\"20200331-12\",\"(5)\"),\\\n",
    "            (\"20200413-12\",\"20200414-12\",\"(6)\"),\\\n",
    "            (\"20200421-12\",\"20200422-12\",\"(7)\"),\\\n",
    "            (\"20200428-12\",\"20200429-12\",\"(8)\"),\\\n",
    "            (\"20200430-12\",\"20200501-12\",\"(9)\"),\\\n",
    "            (\"20200511-12\",\"20200512-12\",\"(10)\"),\\\n",
    "            (\"20200526-12\",\"20200527-12\",\"(11)\"),\\\n",
    "            (\"20200601-12\",\"20200603-12\",\"(12)\"),\\\n",
    "            (\"20200608-12\",\"20200609-12\",\"(13)\"),\\\n",
    "            (\"20200622-12\",\"20200623-12\",\"(14)\"),\\\n",
    "            (\"20200708-12\",\"20200709-12\",\"(15)\"),\n",
    "            (\"20200922-12\",\"20200923-12\",\"(16)\"),\n",
    "            (\"20201006-12\",\"20201007-12\",\"(17)\"),\n",
    "            (\"20201016-12\",\"20201017-12\",\"(18)\"),\n",
    "            (\"20201029-12\",\"20201030-12\",\"(19)\"),\n",
    "            (\"20201214-12\",\"20201216-12\",\"(20)\"),\n",
    "            (\"20201225-12\",\"20201227-12\",\"(21)\"),\n",
    "            (\"20210101-12\",\"20210103-12\",\"(22)\"),\n",
    "            (\"20210124-12\",\"20210126-12\",\"(23)\"),\n",
    "            (\"20210216-12\",\"20210218-12\",\"(24)\"),\n",
    "           ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sentimentPerHour = getSentimentPerHour(DATADIRECTORYALL)\n",
    "sentimentPerDay = makeSentimentPerDay(sentimentPerHour)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dummy = visualizeSentiment([{DATA:{date:sentimentPerDay[date] for date in sentimentPerDay \n",
    "                                                              if date >= \"20200201\"}, LABEL:\"per day\",\\\n",
    "                             HIGHLIGHT:highlight,HIGHLIGHTLABEL:\"press conference\",\n",
    "                             ANNOTATE:[]}],\\\n",
    "                           title=\"Daily average sentiment scores of tweets written in Dutch\",\n",
    "                           tuple_data_list = [\n",
    "                                              [\"20200314-12\", 0.059, \"12-03-2020\\nWork at home\"],\n",
    "                                              [\"20200602-12\", 0.053, \"01-06-2020\\nBLM demonstration Dam\"],\n",
    "                                              [\"20201007-12\", 0.058, \"06-10-2020\\nEddie Van Halen dies\"],\n",
    "                                              [\"20201216-12\", 0.0635, \"14-12-2020\\nLockdown\"],\n",
    "                                              [\"20201208-12\", 0.126, \"25-12\"],\n",
    "                                              [\"20210102-12\", 0.1335, \"01-01\"],\n",
    "                                              [\"20201223-12\", 0.054, \"25-01-2021\\nRiots\"],\n",
    "                                             ])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dummy = visualizeSentiment([{DATA:sentimentPerDay,LABEL:\"per day\",\\\n",
    "                             HIGHLIGHT:highlight,HIGHLIGHTLABEL:\"press conference\",\n",
    "                             ANNOTATE:annotate}],\\\n",
    "                           title=DEFAULTTITLE+\" (all Dutch tweets)\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "```\n",
    " (1) & 20200220 & terrorist attack Hanau, Germany \\\\\n",
    " (2) & 20200302 & newsitem: Netherlands financially supporting asylum seekers \\\\\n",
    " (3) & 20200312 & announcement of first national COVID-19 measures \\\\\n",
    " (4) & 20200322 & lack of social distancing in nature areas \\\\\n",
    " (5) & 20200330 & news item: EU donates millions to Morocco \\\\\n",
    " (6) & 20200413 & news item: anti-gay violence \\\\\n",
    " (7) & 20200421 & national press conference on COVID-19 \\\\\n",
    " (8) & 20200428 & news item: lack of social distancing at IKEA \\\\\n",
    " (9) & 20200430 & news item: local coalition of CDA and FvD \\\\\n",
    "(10) & 20200511 & news item: etnic profiling by tax services \\\\\n",
    "(11) & 20200526 & news item: KLM boss receives bonus \\\\\n",
    "(12) & 20200601 & lack of social distancing at Amsterdam BLM demonstration \\\\\n",
    "(13) & 20200608 & attacks on left-wing politicians \\\\\n",
    "(14) & 20200622 & racism in tv programme Veronica Inside \\\\\n",
    "(15) & 20200708 & farmers protest \\\\\n",
    "(16) & 20200922 & Dutch VIPs involved in protest against measures (#ikdoenietmeermee) \\\\\n",
    "(17) & 20201006 & Eddie Van Halen dies \\\\\n",
    "(18) & 20201016 & Paris teacher killed \\\\\n",
    "(19) & 20201029 & Nice terror attack \\\\\n",
    "(20) & 20201214 & Lockdown starts \\\\\n",
    "(21) & 20201225 & Christmas \\\\\n",
    "(22) & 20210101 & New Year \\\\\n",
    "(23) & 20210124 & Curfew riots \\\\\n",
    "(24) & 20210216 & Judge deems curfew illegal \\\\\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Find dates of sentiment minima with next code block and then look for relevant topics in tweets with these shell commands:\n",
    "```\n",
    "gunzip -c ../sentiment/pattern/20200820-* | cut -d, -f2 > ~/tmp/scores\n",
    "gunzip -c 20200820-* | python3 ../../scripts/get-text.py | grep -v '^text$' | paste -d , ~/tmp/scores - |\\\n",
    "   sort | uniq -c | sort -nr | less\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sortDatesBySentiment(sentimentPerDay,month=\"\"):\n",
    "    return([(hourString,sentimentPerDay[hourString]) \\\n",
    "             for hourString in sorted(sentimentPerDay.keys(),key=lambda h:sentimentPerDay[h][SENTIMENT]) \n",
    "             if re.search(month,hourString)])\n",
    "\n",
    "sortDatesBySentiment(sentimentPerDay,month=\"202010\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Get sentiment on topics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "DATADIRECTORYSENTTOPIC = \"../data/sentiment/topic/\"\n",
    "\n",
    "def getSentimentPerHourQuery(dataDirectory,query,filePattern=DEFAULTFILEPATTERN,computeMissing=True,\\\n",
    "                             dataDirectorySentTopic=DATADIRECTORYSENTTOPIC,dataDirectoryText=DATADIRECTORYTEXT,\n",
    "                             missingPattern=DEFAULTFILEPATTERN):\n",
    "    fileList = sorted(os.listdir(dataDirectory))\n",
    "    sentimentPerHour = {}\n",
    "    for inFileName in fileList:\n",
    "        if re.search(filePattern,inFileName):\n",
    "            if re.search(\"00.out.gz\",inFileName): squeal(inFileName)\n",
    "            if os.path.exists(dataDirectorySentTopic+inFileName):\n",
    "                dfSent = pd.read_csv(dataDirectorySentTopic+inFileName,header=None)\n",
    "                sentiment = sum(dfSent[1])/len(dfSent)\n",
    "                hour = inFileName[0:11]\n",
    "                sentimentPerHour[hour] = { SENTIMENT:sentiment, COUNT:len(dfSent) }\n",
    "            elif computeMissing and re.search(missingPattern,inFileName):\n",
    "                pd.DataFrame([{0:1}]).T.to_csv(dataDirectorySentTopic+inFileName,header=None,index=\"0\")\n",
    "                try:\n",
    "                    dfSent = pd.read_csv(dataDirectory+inFileName,header=None,index_col=0, engine=\"python\")\n",
    "                    dfText = pd.read_csv(dataDirectoryText+inFileName, index_col=\"id_str\", engine=\"python\")\n",
    "                except:\n",
    "                    print(f\"error reading data file {inFileName}!\")\n",
    "                    sys.exit(1)\n",
    "                if len(dfSent) != len(dfText):\n",
    "                    print(f\"lengths of files {inFileName} do not match! {len(dfText)} {len(dfSent)}\")\n",
    "                    sys.exit(1)\n",
    "                matchesText = dfText[dfText[TEXT].str.contains(query,case=False)]\n",
    "                matchesSent = dfSent[dfSent.index.isin(matchesText.index)]\n",
    "                if len(matchesSent) > 0:\n",
    "                    matchesSent.to_csv(dataDirectorySentTopic+inFileName,header=None)\n",
    "                    sentiment = sum(list(matchesSent[1]))/len(matchesSent)\n",
    "                    hour = inFileName[0:11]\n",
    "                    sentimentPerHour[hour] = { SENTIMENT:sentiment, COUNT:len(matchesSent) }\n",
    "                else:\n",
    "                    os.unlink(dataDirectorySentTopic+inFileName)\n",
    "    sentimentPerHour = {key:sentimentPerHour[key] for key in sorted(sentimentPerHour.keys())}\n",
    "    return(sentimentPerHour)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "COMPUTEMISSING = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "TOPICQUERY = \"corona|covid|huisarts|mondkapje|rivm|blijfthuis|flattenthecurve|houvol\"\n",
    "PANDEMICQUERYBASE = \"|\".join([TOPICQUERY, r'virus|besmet|ziekenhui|\\bic\\b|intensive.care|^zorg|vaccin|[^ad]arts|uitbraak|uitbrak|pandemie|ggd|'+\n",
    "                                      r'mondkapje|quarantaine|\\bwho\\b|avondklok|variant|verple|sympto|e.golf|mutant|^omt$|umc|hcq|'+\n",
    "                                      r'hydroxychloroquine|virolo|zkh|oversterfte|patiënt|patient|intensivist|🦠|ivermectin'])\n",
    "DISTANCEQUERY = \"1[.,]5[ -]*m|afstand.*hou|hou.*afstand|anderhalve[ -]*meter\"\n",
    "LOCKDOWNQUERY = \"lock.down|lockdown\"\n",
    "VACCINQUERY = \"vaccin|ingeënt|ingeent|inent|prik|spuit|bijwerking|-->|💉|pfizer|moderna|astrazeneca|astra|zeneca|novavax|biontech\"\n",
    "TESTQUERY = r'\\btest|getest|sneltest|pcr'\n",
    "\n",
    "PANDEMICQUERY = \"|\".join([ DISTANCEQUERY, LOCKDOWNQUERY, PANDEMICQUERYBASE, TESTQUERY, VACCINQUERY])\n",
    "\n",
    "\n",
    "IKQUERY = r'\\b(ik|mij|mijn|me|mn|m\\'n|zelf|mezelf|mijzelf|i)\\b'\n",
    "HAPPYQUERY = r'\\b(geluk|gelukkig|gelukkige|blij|happy)\\b'\n",
    "LONELYQUERY = r'eenza|alleen.*voel|voel.*alleen|lonely|loneli'\n",
    "SWEARQUERY = (r'shit|fuck|klote|kanker|kut|lul|dom|reet|gvd|strot|waanzin|nep|rotzooi|idio|schaamtelo|fake|strot|'+\n",
    "              r'zeik|stom|onbeschoft|oprot|bekrompen|ongelo|bah|kwalijk|onbekwa|achterlijk|jat|schand|triest|puinho|'+\n",
    "              r'immore|kleptocratie|nepotism|absurd|misselijkma|\\bbek\\b|\\brot\\b|\\bkk\\b')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "QUERYTOPIC = \"corona|covid|mondkapje|rivm|blijfthuis|houvol|huisarts|flattenthecurve\"\n",
    "\n",
    "sentimentPerHourTopic = getSentimentPerHourQuery(DATADIRECTORYALL,QUERYTOPIC,filePattern=\"^202\",\\\n",
    "                                                 dataDirectorySentTopic=f\"../data/{DATASOURCE}/sentiment-topic/\",\\\n",
    "                                                 dataDirectoryText=DATADIRECTORYTEXT,computeMissing=COMPUTEMISSING)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "QUERYTOPIC = \"mondkapje\"\n",
    "\n",
    "sentimentPerHourMondkapje = getSentimentPerHourQuery(DATADIRECTORYALL,QUERYTOPIC,filePattern=\"^202\",\\\n",
    "                                                     dataDirectorySentTopic=f\"../data/{DATASOURCE}/sentiment-mondkapje/\",\\\n",
    "                                                     dataDirectoryText=DATADIRECTORYTEXT,computeMissing=COMPUTEMISSING,\n",
    "                                                     missingPattern=\"^20210[2-9]\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "QUERYTOPIC = \"1[.,]5[ -]*m|afstand.*hou|hou.*afstand|anderhalve[ -]*meter\"\n",
    "\n",
    "sentimentPerHourDistance = getSentimentPerHourQuery(DATADIRECTORYALL,QUERYTOPIC,filePattern=\"^202\",\\\n",
    "                                                    dataDirectorySentTopic=f\"../data/{DATASOURCE}/sentiment-distance/\",\\\n",
    "                                                    dataDirectoryText=DATADIRECTORYTEXT,computeMissing=COMPUTEMISSING,\n",
    "                                                    missingPattern=\"^20210[2-9]\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "QUERYTOPIC = \"lockdown|lock.down\"\n",
    "\n",
    "sentimentPerHourLockdown = getSentimentPerHourQuery(DATADIRECTORYALL, QUERYTOPIC, filePattern=\"^202\",\n",
    "                                                    dataDirectorySentTopic=f\"../data/{DATASOURCE}/sentiment-lockdown/\",\n",
    "                                                    dataDirectoryText=DATADIRECTORYTEXT, computeMissing=COMPUTEMISSING,\n",
    "                                                    missingPattern=\"^20210[2-9]\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "QUERYTOPIC = \"vaccin\"\n",
    "\n",
    "sentimentPerHourVaccin = getSentimentPerHourQuery(DATADIRECTORYALL, QUERYTOPIC,filePattern=\"2020*\",\\\n",
    "                                                  dataDirectorySentTopic=f\"../data/{DATASOURCE}/sentiment-vaccin/\",\\\n",
    "                                                  dataDirectoryText=DATADIRECTORYTEXT, computeMissing=COMPUTEMISSING,\n",
    "                                                  missingPattern=\"^20210[2-9]\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "QUERYTOPIC = \"avondklok\"\n",
    "DIRTOPIC = f\"../data/{DATASOURCE}/sentiment-avondklok/\"\n",
    "\n",
    "if not os.path.isdir(DIRTOPIC):\n",
    "    os.mkdir(DIRTOPIC)\n",
    "sentimentPerHourCurfew = getSentimentPerHourQuery(DATADIRECTORYALL, \n",
    "                                                  QUERYTOPIC,\n",
    "                                                  filePattern=\"^202\",\n",
    "                                                  dataDirectorySentTopic=DIRTOPIC,\n",
    "                                                  dataDirectoryText=DATADIRECTORYTEXT,\n",
    "                                                  computeMissing=COMPUTEMISSING,\n",
    "                                                  missingPattern=\"^20210[2-9]\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "QUERYTOPIC = ('shit|fuck|klote|kut|lul|dom|reet|gvd|strot|waanzin|nep|rotzooi|idio|schaamtelo|fake|strot|'+\n",
    "    'zeik|stom|onbeschoft|oprot|bekrompen|ongelo|bah|kwalijk|onbekwa|achterlijk|jat|schand|triest|puinho|'+\n",
    "    'immore|kleptocratie|nepotism|absurd|\\sbek\\s|\\srot\\s|\\skk\\s')\n",
    "DIRTOPIC = f\"../data/{DATASOURCE}/sentiment-swear/\"\n",
    "\n",
    "if not os.path.isdir(DIRTOPIC):\n",
    "    os.mkdir(DIRTOPIC)\n",
    "sentimentPerHourSwear = getSentimentPerHourQuery(DATADIRECTORYALL, \n",
    "                                                 QUERYTOPIC,\n",
    "                                                 filePattern=\"^202\",\n",
    "                                                 dataDirectorySentTopic=DIRTOPIC,\n",
    "                                                 dataDirectoryText=DATADIRECTORYTEXT,\n",
    "                                                 computeMissing=COMPUTEMISSING,\n",
    "                                                 missingPattern=\"^20210[2-9]\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "QUERYTOPIC = r'\\btest|getest|sneltest|pcr'\n",
    "DIRTOPIC = f\"../data/{DATASOURCE}/testing/\"\n",
    "\n",
    "if not os.path.isdir(DIRTOPIC):\n",
    "    os.mkdir(DIRTOPIC)\n",
    "sentimentPerHourTest = getSentimentPerHourQuery(DATADIRECTORYALL, \n",
    "                                                QUERYTOPIC,\n",
    "                                                filePattern=\"^202\",\n",
    "                                                dataDirectorySentTopic=DIRTOPIC,\n",
    "                                                dataDirectoryText=DATADIRECTORYTEXT,\n",
    "                                                computeMissing=COMPUTEMISSING,\n",
    "                                                missingPattern=\"^202\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "QUERYTOPIC = PANDEMICQUERY\n",
    "DIRTOPIC = f\"../data/{DATASOURCE}/sentiment-pandemic/\"\n",
    "\n",
    "if not os.path.isdir(DIRTOPIC):\n",
    "    os.mkdir(DIRTOPIC)\n",
    "sentimentPerHourPandemic = getSentimentPerHourQuery(DATADIRECTORYALL, \n",
    "                                                    QUERYTOPIC,\n",
    "                                                    filePattern=\"^202\",\n",
    "                                                    dataDirectorySentTopic=DIRTOPIC,\n",
    "                                                    dataDirectoryText=DATADIRECTORYTEXT,\n",
    "                                                    computeMissing=COMPUTEMISSING,\n",
    "                                                    missingPattern=\"^202\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sentimentPerDayMondkapje = makeSentimentPerDay(sentimentPerHourMondkapje)\n",
    "sentimentPerDayDistance = makeSentimentPerDay(sentimentPerHourDistance)\n",
    "sentimentPerDayTopic = makeSentimentPerDay(sentimentPerHourTopic)\n",
    "sentimentPerDayLockdown = makeSentimentPerDay(sentimentPerHourLockdown)\n",
    "sentimentPerDayVaccin = makeSentimentPerDay(sentimentPerHourVaccin)\n",
    "sentimentPerDayCurfew = makeSentimentPerDay(sentimentPerHourCurfew)\n",
    "sentimentPerDaySwear = makeSentimentPerDay(sentimentPerHourSwear)\n",
    "sentimentPerDayPandemic = makeSentimentPerDay(sentimentPerHourPandemic)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "FIRSTCOVIDDAY = \"20200312-12\"\n",
    "\n",
    "sentimentPerDayMondkapje = {key:sentimentPerDayMondkapje[key] for key in sentimentPerDayMondkapje.keys() if key >= FIRSTCOVIDDAY}\n",
    "sentimentPerDayDistance = {key:sentimentPerDayDistance[key] for key in sentimentPerDayDistance.keys() if key >= FIRSTCOVIDDAY}\n",
    "sentimentPerDayLockdown = {key:sentimentPerDayLockdown[key] for key in sentimentPerDayLockdown.keys() if key >= FIRSTCOVIDDAY}\n",
    "sentimentPerDayVaccin = {key:sentimentPerDayVaccin[key] for key in sentimentPerDayVaccin.keys() if key >= FIRSTCOVIDDAY}\n",
    "sentimentPerDayCurfew = {key:sentimentPerDayCurfew[key] for key in sentimentPerDayCurfew.keys() if key >= FIRSTCOVIDDAY}\n",
    "sentimentPerDaySwear = {key:sentimentPerDaySwear[key] for key in sentimentPerDaySwear.keys() if key >= FIRSTCOVIDDAY}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "(len(sentimentPerDayMondkapje), len(sentimentPerDayDistance), len(sentimentPerDayLockdown), len(sentimentPerDayVaccin),\n",
    " len(sentimentPerDayCurfew), len(sentimentPerDaySwear), len(sentimentPerDayPandemic), len(sentimentPerDayTopic), len(sentimentPerDay))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## General graphs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "annotateTopic = [(\"20200215-12\",\"20200216-12\",\"(A)\"),\n",
    "                 (\"20200223-12\",\"20200224-12\",\"(B)\"),\n",
    "                 (\"20200308-12\",\"20200309-12\",\"(C)\"),\n",
    "                 (\"20200322-12\",\"20200323-12\",\"(D)\"),\n",
    "                 (\"20200411-12\",\"20200412-12\",\"(E)\"),\n",
    "                 (\"20200420-12\",\"20200421-12\",\"(F)\"),\n",
    "                 (\"20200510-12\",\"20200511-12\",\"(G)\"),\n",
    "                 (\"20200519-12\",\"20200520-12\",\"(H)\"),\n",
    "                 (\"20200601-12\",\"20200602-12\",\"(I)\"),\n",
    "                 (\"20200621-12\",\"20200622-12\",\"(J)\"),\n",
    "                 (\"20200712-12\",\"20200713-12\",\"(K)\"),\n",
    "                 (\"20200716-12\",\"20200717-12\",\"(L)\"),\n",
    "                 (\"20200813-12\",\"20200814-12\",\"(M)\"),\n",
    "                 (\"20200901-12\",\"20200902-12\",\"(N)\"),\n",
    "                 (\"20210124-12\",\"20210125-12\",\"(O)\"),\n",
    "                ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "PLOTFILENAME = \"sentiment-all.png\"\n",
    "STARTDATE = \"20200215-00\"\n",
    "\n",
    "dummy = visualizeSentiment([{DATA:{date:sentimentPerDay[date] for date in sentimentPerDay if date >= STARTDATE}, \n",
    "                             LABEL:\"all tweets\",\n",
    "                             HIGHLIGHT:highlight,\n",
    "                             ANNOTATE:annotate},\n",
    "                            {DATA:{date:sentimentPerDayPandemic[date] for date in sentimentPerDayPandemic if date >= STARTDATE},\n",
    "                             LABEL:\"pandemic tweets\",\n",
    "                             HIGHLIGHT:highlight,\n",
    "                             HIGHLIGHTLABEL:\"press conference\",\n",
    "                             ANNOTATE:annotateTopic}],\n",
    "                           title=DEFAULTTITLE+\" (all Dutch tweets vs topic tweets)\",outFileName=PLOTFILENAME)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "```\n",
    "(A) & 20200215 & COVID-19 death on Dutch cruiseship \\\\\n",
    "(B) & 20200223 & Italy outbreak \\\\\n",
    "(C) & 20200308 & news item: COVID-19 spreading in Europe \\\\\n",
    "(D) & 20200322 & news item: lack of social distancing in nature areas \\\\\n",
    "(E) & 20200411 & news item: Netherlands sent facemasks to China \\\\\n",
    "(F) & 20200420 & anti-climate protest on Twitter \\\\\n",
    "(G) & 20200510 & warnings about staying alert \\\\\n",
    "(H) & 20200519 & relaxation of COVID-19 measures announced \\\\\n",
    "(I) & 20200601 & lack of social distancing at Amsterdam BLM demonstration \\\\\n",
    "(J) & 20200621 & The Hague demonstration cancelled for lack of social distancing \\\\\n",
    "(K) & 20200712 & news item: facemask related murder in France \\\\\n",
    "(L) & 20200716 & discussion about initial health care worker safety \\\\\n",
    "(M) & 20200813 & coalition leaves parliament to avoid vote on health care worker salaries \\\\\n",
    "(N) & 20200902 & new photos showing lack of social distancing wedding Grapperhaus \\\\\n",
    "(O) & 20210124 & Curfew riots \\\\\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#sortDatesBySentiment(sentimentPerDay,month=\"202103\")\n",
    "sortDatesBySentiment(sentimentPerDayPandemic,month=\"202101\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_extreme_tweets(date, topic, negative=False):\n",
    "    df = {}\n",
    "    for hour in range(0,24):\n",
    "        hour = str(hour).zfill(2)\n",
    "        file = f\"{date}-{hour}.out.gz\"\n",
    "        tweets = pd.read_csv(DATADIRECTORYTEXT+file, index_col=\"id_str\")\n",
    "        sentiment = pd.read_csv(DATADIRECTORYALL+file, header=None, index_col=0)\n",
    "        sentiment = sentiment.rename(columns={1:\"sentiment\"}, index={0:\"id_str\"})\n",
    "        tweets_sent = tweets.join(sentiment)[[\"text\",\"sentiment\"]]\n",
    "        if topic != \"\":\n",
    "            tweets_sent = tweets_sent[tweets_sent[\"text\"].str.contains(topic)]\n",
    "        if len(df) == 0: \n",
    "            df = tweets_sent\n",
    "        else:\n",
    "            df = pd.concat([df,tweets_sent])\n",
    "    groups = df.groupby([\"text\",\"sentiment\"]).groups\n",
    "    if negative:\n",
    "        return([(len(groups[group]),group) for group in sorted(groups, key=lambda group:len(groups[group]), reverse=True) if group[1] < 0])\n",
    "    else:\n",
    "        return([(len(groups[group]),group) for group in sorted(groups, key=lambda group:len(groups[group]), reverse=True) if group[1] > 0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "QUERYTOPIC = \"corona|covid|mondkapje|rivm|blijfthuis|houvol|huisarts|flattenthecurve\"\n",
    "ALLTWEETSTOPIC = \"\"\n",
    "\n",
    "get_extreme_tweets(\"20210212\", QUERYTOPIC, negative=True)[:20]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Graph for EMNLP paper"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "annotatePaper = [(\"20200312-12\",\"20200313-12\",\"(A)\"),\n",
    "                 (\"20200519-12\",\"20200520-12\",\"(B)\")]\n",
    "\n",
    "dummy = visualizeSentiment([{DATA:sentimentPerDay,LABEL:\"all tweets\",\n",
    "                             HIGHLIGHT:[],HIGHLIGHTLABEL:\"\",ANNOTATE:[]},\n",
    "                            {DATA:sentimentPerDayTopic,LABEL:\"topic tweets\",HIGHLIGHT:[],\n",
    "                             ANNOTATE:annotatePaper}],\n",
    "                           title=DEFAULTTITLE+\" (all Dutch tweets vs topic tweets)\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def show_dates_higher_covid_sentiment():\n",
    "    print([(d,sentimentPerDay[d][SENTIMENT],sentimentPerDayTopic[d][SENTIMENT]) for d in sentimentPerDayTopic \\\n",
    "           if sentimentPerDayTopic[d][SENTIMENT] > sentimentPerDay[d][SENTIMENT]])\n",
    "\n",
    "show_dates_higher_covid_sentiment()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "[(hourString,sentimentPerDayTopic[hourString]) \\\n",
    " for hourString in sorted(sentimentPerDayTopic.keys(),key=lambda h:sentimentPerDayTopic[h][SENTIMENT]) if re.search(\"202010\",hourString)]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Two per-topic graphs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "highlightPaper = [\"20200312-12\",\"20200519-12\"]\n",
    "annotatePaper = [(\"20200312-12\",\"20200313-12\",\"(A)\"),\n",
    "                 (\"20200519-12\",\"20200520-12\",\"(B)\")]\n",
    "STARTDATE = \"20210101-12\"\n",
    "\n",
    "dummy = visualizeSentiment([\n",
    "            {DATA:sentimentPerDay,LABEL:\"all\",HIGHLIGHT:[],HIGHLIGHTLABEL:\"\",ANNOTATE:[]},\n",
    "            {DATA:sentimentPerDayPandemic,LABEL:\"COVID-19\",HIGHLIGHT:highlightPaper,HIGHLIGHTLABEL:\"\",ANNOTATE:annotatePaper}],\n",
    "            title=DEFAULTTITLE)\n",
    "dummy = visualizeSentiment([\n",
    "            {DATA:{k:sentimentPerDayMondkapje[k] for k in sentimentPerDayMondkapje if k >= STARTDATE},LABEL:\"face masks\",HIGHLIGHT:\"\",HIGHLIGHTLABEL:\"\",ANNOTATE:\"\"},\n",
    "            {DATA:{k:sentimentPerDayDistance[k] for k in sentimentPerDayDistance if k >= STARTDATE},LABEL:\"distancing\",HIGHLIGHT:\"\",HIGHLIGHTLABEL:\"\",ANNOTATE:\"\"},\n",
    "            {DATA:{k:sentimentPerDayLockdown[k] for k in sentimentPerDayLockdown if k >= STARTDATE},LABEL:\"lockdown\",HIGHLIGHT:\"\",HIGHLIGHTLABEL:\"\",ANNOTATE:\"\"},\n",
    "            {DATA:{k:sentimentPerDayVaccin[k] for k in sentimentPerDayVaccin if k >= STARTDATE},LABEL:\"vaccin\",HIGHLIGHT:\"\",HIGHLIGHTLABEL:\"\",ANNOTATE:\"\"},\n",
    "            {DATA:{k:sentimentPerDayCurfew[k] for k in sentimentPerDayCurfew if k >= STARTDATE},LABEL:\"curfew\",HIGHLIGHT:\"\",HIGHLIGHTLABEL:\"\",ANNOTATE:\"\"},\n",
    "            {DATA:{k:sentimentPerDaySwear[k] for k in sentimentPerDaySwear if k >= STARTDATE},LABEL:\"swear\",HIGHLIGHT:\"\",HIGHLIGHTLABEL:\"\",ANNOTATE:\"\"},\n",
    "            {DATA:{k:sentimentPerDayPandemic[k] for k in sentimentPerDayPandemic if k >= STARTDATE},LABEL:\"pandemic\",HIGHLIGHT:\"\",HIGHLIGHTLABEL:\"\",ANNOTATE:\"\"},\n",
    "            ],\n",
    "            title=DEFAULTTITLE+\" (per day)\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def movingAverage(sentiment_per_day,size):\n",
    "    sentiment_per_day_items = list(sentiment_per_day.items())\n",
    "    new_sentiment_per_day_items = []\n",
    "    for i in range(0,len(sentiment_per_day_items)):\n",
    "        new_sentiment_per_day_items.append((sentiment_per_day_items[i][0],{SENTIMENT:0, COUNT:0}))\n",
    "        for j in range(i-size+1+round(0.5*size),i+1+round(0.5*size)):\n",
    "            if j >= 0 and j < len(sentiment_per_day):\n",
    "                new_sentiment_per_day_items[-1][1][SENTIMENT] += sentiment_per_day_items[j][1][SENTIMENT]*sentiment_per_day_items[j][1][COUNT]\n",
    "                new_sentiment_per_day_items[-1][1][COUNT] += sentiment_per_day_items[j][1][COUNT]\n",
    "        new_sentiment_per_day_items[-1][1][SENTIMENT] /= new_sentiment_per_day_items[-1][1][COUNT]\n",
    "    return(dict(new_sentiment_per_day_items))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def movingMedian(sentiment_per_day, size):\n",
    "    sentiment_per_day_items = list(sentiment_per_day.items())\n",
    "    new_sentiment_per_day_items = []\n",
    "    for i in range(0,len(sentiment_per_day_items)):\n",
    "        new_sentiment_per_day_items.append([sentiment_per_day_items[i][0], []])\n",
    "        for j in range(i-size,i+1+size):\n",
    "            if j >= 0 and j < len(sentiment_per_day):\n",
    "                new_sentiment_per_day_items[-1][1].append(sentiment_per_day_items[j][1][SENTIMENT])\n",
    "        new_sentiment_per_day_items[-1][1] = { SENTIMENT:statistics.median(new_sentiment_per_day_items[-1][1]), \n",
    "                                               COUNT: sentiment_per_day_items[i][1][COUNT] }\n",
    "    return(dict(new_sentiment_per_day_items))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pretty_number(number):\n",
    "    digits = str(number)\n",
    "    pretty_digits = \"\"\n",
    "    for i in range(0,len(digits)):\n",
    "        if i > 0 and i % 3 == 0: pretty_digits = \",\"+pretty_digits\n",
    "        pretty_digits = digits[-1-i]+pretty_digits\n",
    "    return(pretty_digits)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "NBROFDAYS = 60\n",
    "EXTRASMOOTHDAYS = 30\n",
    "MEDIANCONTEXT = 14\n",
    "\n",
    "#mondkapjeData = movingAverage(movingAverage(sentimentPerDayMondkapje, NBROFDAYS), EXTRASMOOTHDAYS)\n",
    "#distanceData = movingAverage(movingAverage(sentimentPerDayDistance, NBROFDAYS), EXTRASMOOTHDAYS)\n",
    "#lockdownData = movingAverage(movingAverage(sentimentPerDayLockdown, NBROFDAYS), EXTRASMOOTHDAYS)\n",
    "#vaccinData = movingAverage(movingAverage(sentimentPerDayVaccin, NBROFDAYS), EXTRASMOOTHDAYS)\n",
    "#curfewData = movingAverage(movingAverage(sentimentPerDayCurfew, NBROFDAYS), EXTRASMOOTHDAYS)\n",
    "#swearData = movingAverage(movingAverage(sentimentPerDaySwear, NBROFDAYS), EXTRASMOOTHDAYS)\n",
    "\n",
    "mondkapjeData = movingMedian(sentimentPerDayMondkapje, MEDIANCONTEXT)\n",
    "distanceData = movingMedian(sentimentPerDayDistance, MEDIANCONTEXT)\n",
    "lockdownData = movingMedian(sentimentPerDayLockdown, MEDIANCONTEXT)\n",
    "vaccinData = movingMedian(sentimentPerDayVaccin, MEDIANCONTEXT)\n",
    "curfewData = movingMedian(sentimentPerDayCurfew, MEDIANCONTEXT)\n",
    "swearData = movingMedian(sentimentPerDaySwear, MEDIANCONTEXT)\n",
    "pandemicData = movingMedian(sentimentPerDayPandemic, MEDIANCONTEXT)\n",
    "\n",
    "mondkapjeDayCounts = np.sum([sentimentPerDayMondkapje[day][\"count\"] for day in sentimentPerDayMondkapje])\n",
    "distanceDayCounts = np.sum([sentimentPerDayDistance[day][\"count\"] for day in sentimentPerDayDistance])\n",
    "lockdownDayCounts = np.sum([sentimentPerDayLockdown[day][\"count\"] for day in sentimentPerDayLockdown])\n",
    "vaccinDayCounts = np.sum([sentimentPerDayVaccin[day][\"count\"] for day in sentimentPerDayVaccin])\n",
    "curfewDayCounts = np.sum([sentimentPerDayCurfew[day][\"count\"] for day in sentimentPerDayCurfew])\n",
    "swearDayCounts = np.sum([sentimentPerDaySwear[day][\"count\"] for day in sentimentPerDaySwear])\n",
    "pandemicDayCounts = np.sum([sentimentPerDayPandemic[day][\"count\"] for day in sentimentPerDayPandemic])\n",
    "\n",
    "dummy = visualizeSentiment([\n",
    "            {DATA:mondkapjeData, LABEL:f\"face masks ({pretty_number(mondkapjeDayCounts)})\", HIGHLIGHT:\"\", HIGHLIGHTLABEL:\"\", ANNOTATE:\"\"},\n",
    "            {DATA:lockdownData, LABEL:f\"lockdown ({pretty_number(lockdownDayCounts)})\", HIGHLIGHT:\"\", HIGHLIGHTLABEL:\"\", ANNOTATE:\"\"},\n",
    "            {DATA:distanceData, LABEL:f\"distancing ({pretty_number(distanceDayCounts)})\", HIGHLIGHT:\"\", HIGHLIGHTLABEL:\"\", ANNOTATE:\"\"},\n",
    "            {DATA:vaccinData, LABEL:f\"vaccination ({pretty_number(vaccinDayCounts)})\", HIGHLIGHT:\"\", HIGHLIGHTLABEL:\"\", ANNOTATE:\"\"},\n",
    "            {DATA:curfewData, LABEL:f\"curfew ({pretty_number(curfewDayCounts)})\", HIGHLIGHT:\"\", HIGHLIGHTLABEL:\"\", ANNOTATE:\"\"},\n",
    "            {DATA:swearData, LABEL:f\"swear ({pretty_number(swearDayCounts)})\", HIGHLIGHT:\"\", HIGHLIGHTLABEL:\"\", ANNOTATE:\"\"},\n",
    "            {DATA:pandemicData, LABEL:f\"pandemic ({pretty_number(pandemicDayCounts)})\", HIGHLIGHT:\"\", HIGHLIGHTLABEL:\"\", ANNOTATE:\"\"},\n",
    "        ], title=DEFAULTTITLE+f\" (median over {1+2*MEDIANCONTEXT} days)\",skip_factor=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Two plots in one figure"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "PLOTFILENAME = \"sentiment-all-split.png\"\n",
    "highlightPaperAll = [\"20200312-12\"]\n",
    "annotatePaperAll = [(\"20200312-12\",\"20200313-12\",\"(A)\")]\n",
    "highlightPaperCovid = [\"20200519-12\"]\n",
    "annotatePaperCovid = [(\"20200519-12\",\"20200520-12\",\"(B)\")]\n",
    "\n",
    "dateFormat = DATEFORMATMONTH\n",
    "dataSources = [{DATA:sentimentPerDay,LABEL:\"all\",HIGHLIGHT:highlightPaperAll,HIGHLIGHTLABEL:\"\",ANNOTATE:annotatePaperAll},\n",
    "               {DATA:sentimentPerDayPandemic,LABEL:\"COVID-19\",HIGHLIGHT:highlightPaperCovid,HIGHLIGHTLABEL:\"\",ANNOTATE:annotatePaperCovid}]\n",
    "title = DEFAULTTITLE\n",
    "YMIN = -0.13\n",
    "YMAX =0.17\n",
    "\n",
    "if True:\n",
    "    font = {\"size\":12}\n",
    "    matplotlib.rc(\"font\",**font)\n",
    "    fig,ax = plt.subplots(figsize=(12,6))\n",
    "    ax = plt.subplot(121)\n",
    "    plt.ylim(YMIN,YMAX) # was -0.14,0.18\n",
    "    ax.xaxis.set_major_formatter(mdates.DateFormatter(dateFormat))\n",
    "    for i in range(0,len(dataSources)):\n",
    "        data = dataSources[i][DATA]\n",
    "        label = dataSources[i][LABEL]\n",
    "        lineData = ax.plot_date([stringToDate(key) for key in data if data[key][COUNT] > 1],\\\n",
    "                     [data[key][SENTIMENT] for key in data if data[key][COUNT] > 1],xdate=True,fmt=\"-\",label=label)\n",
    "        if HIGHLIGHT in dataSources[i]:\n",
    "            highlight = dataSources[i][HIGHLIGHT]\n",
    "            color = lineData[-1].get_color()\n",
    "            if not HIGHLIGHTLABEL in dataSources[i]:\n",
    "                ax.plot_date([stringToDate(key) for key in highlight],\n",
    "                             [data[key][SENTIMENT] for key in highlight],\\\n",
    "                             fmt=\"o\",color=\"black\")\n",
    "            else:\n",
    "                highlightlabel = dataSources[i][HIGHLIGHTLABEL]\n",
    "                ax.plot_date([stringToDate(key) for key in highlight],\n",
    "                             [data[key][SENTIMENT] for key in highlight],\\\n",
    "                             fmt=\"o\",color=\"black\",label=highlightlabel)\n",
    "        if ANNOTATE in dataSources[i]:\n",
    "            for date1,date2,text in dataSources[i][ANNOTATE]:\n",
    "                plt.annotate(text,(stringToDate(date2),data[date1][SENTIMENT]),color=\"black\")\n",
    "    plt.title(\"                               Polarity scores of\")\n",
    "    plt.xlabel(\"dates from February to October 2020\")\n",
    "    plt.ylabel(\"polarity score (min: -1.0; max: +1.0)\")\n",
    "    plt.legend(framealpha=0.2)\n",
    "\n",
    "dataSources = [{DATA:sentimentPerDayDistance,LABEL:\"distancing\",HIGHLIGHT:highlightPaper,HIGHLIGHTLABEL:\"\",ANNOTATE:annotatePaper},\n",
    "               {DATA:sentimentPerDayMondkapje,LABEL:\"face masks\",HIGHLIGHT:highlightPaper,HIGHLIGHTLABEL:\"\",ANNOTATE:annotatePaper},\n",
    "               {DATA:sentimentPerDayLockdown,LABEL:\"lockdown\",HIGHLIGHT:highlightPaper,HIGHLIGHTLABEL:\"\",ANNOTATE:annotatePaper},\n",
    "               {DATA:sentimentPerDayVaccin,LABEL:\"vaccin\",HIGHLIGHT:highlightPaper,HIGHLIGHTLABEL:\"\",ANNOTATE:annotatePaper},\n",
    "               {DATA:sentimentPerDayCurfew,LABEL:\"curfew\",HIGHLIGHT:highlightPaper,HIGHLIGHTLABEL:\"\",ANNOTATE:annotatePaper},\n",
    "               {DATA:sentimentPerDaySwear,LABEL:\"swear\",HIGHLIGHT:highlightPaper,HIGHLIGHTLABEL:\"\",ANNOTATE:annotatePaper},\n",
    "               {DATA:sentimentPerDayPandemic,LABEL:\"pandemic\",HIGHLIGHT:highlightPaper,HIGHLIGHTLABEL:\"\",ANNOTATE:annotatePaper},\n",
    "              ]\n",
    "\n",
    "if True:\n",
    "    ax = plt.subplot(122)\n",
    "    plt.ylim(YMIN,YMAX)\n",
    "    ax.xaxis.set_major_formatter(mdates.DateFormatter(dateFormat))\n",
    "    for i in range(0,len(dataSources)):\n",
    "        data = dataSources[i][DATA]\n",
    "        label = dataSources[i][LABEL]\n",
    "        x = [key for key in data if data[key][COUNT] > 1]\n",
    "        y = [data[key][SENTIMENT] for key in data if data[key][COUNT] > 1]\n",
    "        lineData = ax.plot_date([stringToDate(key) for key in x],y,xdate=True,fmt=\"-\",label=label)\n",
    "        plottedData = {x[i]:y[i] for i in range(0,len(x))}\n",
    "        #if HIGHLIGHT in dataSources[i]:\n",
    "        #    highlight = dataSources[i][HIGHLIGHT]\n",
    "        #    color = lineData[-1].get_color()\n",
    "        #    if not HIGHLIGHTLABEL in dataSources[i]:\n",
    "        #        ax.plot_date([stringToDate(key) for key in highlight],\n",
    "        #                     [plottedData[key] for key in highlight],\\\n",
    "        #                     fmt=\"o\",color=color)\n",
    "        #    else:\n",
    "        #        highlightlabel = dataSources[i][HIGHLIGHTLABEL]\n",
    "        #        ax.plot_date([stringToDate(key) for key in highlight],\n",
    "        #                     [plottedData[key] for key in highlight],\\\n",
    "        #                     fmt=\"o\",color=color,label=highlightlabel)\n",
    "        #if ANNOTATE in dataSources[i]:\n",
    "        #    for date1,date2,text in dataSources[i][ANNOTATE]:\n",
    "        #        plt.annotate(text,(stringToDate(date2),plottedData[date1]),color=color)\n",
    "    plt.title(\"Dutch tweets over time                      \")\n",
    "    plt.legend(framealpha=0.2)\n",
    "    plt.xlabel(\"dates from March to October 2020\")\n",
    "    plt.savefig(PLOTFILENAME)\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def averageSentiment(sentimentPerDay):\n",
    "    sentiment = 0.0\n",
    "    count = 0\n",
    "    for date in sentimentPerDay.keys():\n",
    "        sentiment += sentimentPerDay[date][SENTIMENT]*sentimentPerDay[date][COUNT]\n",
    "        count += sentimentPerDay[date][COUNT]\n",
    "    return(sentiment/count)\n",
    "\n",
    "print(averageSentiment(sentimentPerDayMondkapje),\n",
    "      averageSentiment(sentimentPerDayDistance),\n",
    "      averageSentiment(sentimentPerDayLockdown),\n",
    "      averageSentiment(sentimentPerDayVaccin),\n",
    "      averageSentiment(sentimentPerDaySwear),\n",
    "      averageSentiment(sentimentPerDayPandemic),\n",
    "     )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Average sentiment scores:\n",
    "1. Twitter: face masks: 0.037; distancing: 0,064; lockdown: 0.054 (-1 Nov)\n",
    "2. Nu.nl: face masks: 0.055; distancing: 0.056 (-1 Aug)\n",
    "3. Reddit: face masks: 0.050; distancing: 0.068 (-1 Aug)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## STOP HERE"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Show positive tweets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def showTweetTextWithSentiment(dataDir,query,filePattern=DEFAULTFILEPATTERN):\n",
    "    fileList = sorted(os.listdir(dataDir))\n",
    "    tweetSentPairs = []\n",
    "    for inFileName in fileList:\n",
    "        if re.search(filePattern,inFileName):\n",
    "            squeal(inFileName)\n",
    "            if os.path.exists(dataDir+inFileName):\n",
    "                dfSent = pd.read_csv(dataDir+inFileName,header=None,index_col=0)\n",
    "                dfText = pd.read_csv(DATADIRECTORYTEXT+inFileName,index_col=IDSTR)\n",
    "                selectedDict = {idStr:dfText.loc[idStr][TEXT] for idStr in dfText.index if re.search(query,dfText.loc[idStr][TEXT],flags=re.IGNORECASE)}\n",
    "                for idStr in selectedDict.keys():\n",
    "                    try:\n",
    "                        tweetSentPairs.append((dfSent.loc[idStr][1],selectedDict[idStr]))\n",
    "                    except: pass\n",
    "    tweetSentPairs = sorted(tweetSentPairs,key=lambda e:e[0],reverse=True)\n",
    "    return(tweetSentPairs)\n",
    "\n",
    "tweetSentPairs = showTweetTextWithSentiment(DATADIRECTORYSENTTOPIC,query=\"coronapocalypse\",filePattern=\"20200316\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tweetSentPairs[0:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sum([tweetSentPairs[i][0] for i in range(0,len(tweetSentPairs))])/len(tweetSentPairs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Visualize sentiment of single press conference day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def convertDate(dateString1,dateString2,dateString3):\n",
    "    return(re.sub(dateString1,dateString2,dateString3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dummy = visualizeSentiment([{DATA:{k:sentimentPerHour[k] for k in sentimentPerHour.keys() \\\n",
    "                                   if re.search(r\"^20200312\",k)},\\\n",
    "                             LABEL:\"20200312\",HIGHLIGHT:[\"20200312-15\"],HIGHLIGHTLABEL:\"press conference\"},\\\n",
    "                            {DATA:{convertDate(\"20200311\",\"20200312\",k):sentimentPerHour[k] for k in sentimentPerHour.keys() \n",
    "                                   if re.search(r\"^20200311\",k)},LABEL:\"20200311\"},\\\n",
    "                            {DATA:{convertDate(\"20200313\",\"20200312\",k):sentimentPerHour[k] for k in sentimentPerHour.keys() \n",
    "                                   if re.search(r\"^20200313\",k)},LABEL:\"20200313\"}],\\\n",
    "                           title=\"Sentiment scores of Dutch tweets on Thursday 12 March 2020\",dateFormat=DATEFORMATHRSMINS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dummy = visualizeSentiment([{DATA:{k:sentimentPerHour[k] for k in sentimentPerHour.keys() \\\n",
    "                                   if re.search(r\"^20200312\",k)},\\\n",
    "                             LABEL:\"all tweets\",HIGHLIGHT:[\"20200312-15\"],HIGHLIGHTLABEL:\"press conference\"}],\\\n",
    "                           title=\"Polarity scores of Dutch tweets on Thursday 12 March 2020\",dateFormat=DATEFORMATHRSMINS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "{DATA:{k:sentimentPerHour[k] for k in sentimentPerHour.keys() if re.search(r\"^20200311\",k)},LABEL:\"\"}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Compare day graph with previous years"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "DATADIRECTORYALL2019 = \"../data/sentiment/2019/\"\n",
    "TEMPERATURESFILE = \"../data/temperatures.csv\" \n",
    "\n",
    "def changeYearTo2020(dataIn):\n",
    "    dataOut = {}\n",
    "    for dateString in dataIn:\n",
    "        dateString2020 = re.sub(r\"20[0-9][0-9]\",\"2020\",dateString)\n",
    "        dataOut[dateString2020] = dataIn[dateString]\n",
    "    return(dataOut)\n",
    "\n",
    "sentimentPerHour2019 = getSentimentPerHour(DATADIRECTORYALL2019)\n",
    "sentimentPerDay2019 = makeSentimentPerDay(sentimentPerHour2019)\n",
    "ax = visualizeSentiment([{DATA:sentimentPerDay,LABEL:\"2020 per day\",\\\n",
    "                          HIGHLIGHT:highlight,HIGHLIGHTLABEL:\"press conference\"},\\\n",
    "                         {DATA:changeYearTo2020(sentimentPerDay2019),LABEL:\"2019 per day\",\\\n",
    "                          HIGHLIGHT:[\"20200315-12\",\"20200318-12\"],HIGHLIGHTLABEL:\"terror attack\"}],\n",
    "                         title=DEFAULTTITLE)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Add daily temperatures to plot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def readTemperatures():\n",
    "    temperatures = pd.read_csv(TEMPERATURESFILE,header=None)\n",
    "    temperatures = { (str(temperatures[0][i])+\"-12\"):(temperatures[1][i]/10) for i in range(0,len(temperatures))}\n",
    "    return(temperatures)\n",
    "\n",
    "temperatures = readTemperatures()\n",
    "\n",
    "ax2 = ax.twinx()\n",
    "ax2.set_ylim([-20,40])\n",
    "dummy = ax2.plot_date([datetime.datetime.strptime(key,DATEFORMATHOUR) for key in temperatures],\\\n",
    "                      [temperatures[key] for key in temperatures],xdate=True,\\\n",
    "                      fmt=\"--\",color=\"990000\",label=\"temperature\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Check relation temperature - sentiment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(3,3))\n",
    "plt.gcf().subplots_adjust(left=0.2)\n",
    "x = [sentimentPerDay[key][SENTIMENT] for key in temperatures]\n",
    "y = [temperatures[key] for key in temperatures]\n",
    "plt.scatter(x,y)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.corrcoef(x,y)[0][1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Check relation sentiment today - yesterday "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(3,3))\n",
    "plt.gcf().subplots_adjust(left=0.2)\n",
    "x = [sentimentPerDay[key][SENTIMENT] for key in sentimentPerDay]\n",
    "y = [sentimentPerDay[key][SENTIMENT] for key in sentimentPerDay]\n",
    "x.pop(-1)\n",
    "y.pop(0)\n",
    "plt.scatter(x,y)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.corrcoef(x,y)[0][1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Sort days by sentiment score\n",
    "\n",
    "Dates of Dutch government press conferences related to corona crisis:\n",
    "\n",
    "1. Sunday 1 March 2020: https://www.youtube.com/watch?v=rg9aM97Pu38\n",
    "2. Monday 9 March 2020: https://www.youtube.com/watch?v=xz-DtYHGdLA\n",
    "3. Thursday 12 March 2020: https://www.youtube.com/watch?v=0iD1FN6I87Y (start: around 15:15 PM)\n",
    "4. Sunday 15 March 2020: https://www.youtube.com/watch?v=j94ULn90xg8\n",
    "5. Tuesday 17 March 2020: https://www.youtube.com/watch?v=KuXj3c1F8WY\n",
    "6. Thursday 19 March 2020: https://www.youtube.com/watch?v=Yqx0PlSnUFE\n",
    "7. Monday 23 March 2020: https://www.youtube.com/watch?v=mcpLFX_L9o8\n",
    "8. Tuesday 31 March 2020: https://www.youtube.com/watch?v=FNuzw3wplow\n",
    "9. Tuesday 7 April 2020: https://www.youtube.com/watch?v=-j3_mmZcBZU\n",
    "10. Wednesday 15 April 2020: https://www.youtube.com/watch?v=n1-rSb3j0UM\n",
    "11. Tuesday 21 April 2020: https://www.youtube.com/watch?v=Yjx7SPtq7Bk\n",
    "12. Wednesday 29 April 2020: https://www.youtube.com/watch?v=dfhzIGagbOw\n",
    "13. Wednesday 6 May 2020: https://www.youtube.com/watch?v=rBtDphRcPKA\n",
    "14. Wednesday 13 May 2020: https://www.youtube.com/watch?v=zISG9KkuAQ0\n",
    "15. Tuesday 19 May 2020: https://www.youtube.com/watch?v=MlAk_RMTHZU\n",
    "16. Wednesday 27 May 2020: https://www.youtube.com/watch?v=M0qBpBz_5h8\n",
    "17. Wednesday 3 June 2020: https://www.youtube.com/watch?v=rsIUKfLjf8k"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "{k:round(v[SENTIMENT],4) for k,v in sorted(sentimentPerDay.items(),key=lambda item:item[1][SENTIMENT])}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Process data from RIVM query"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sentimentPerHour = getSentimentPerHour(DATADIRECTORYRIVM)\n",
    "sentimentPerDay = makeSentimentPerDay(sentimentPerHour)\n",
    "visualizeSentiment([{DATA:sentimentPerHour,LABEL:\"per hour\"},{DATA:sentimentPerDay,LABEL:\"per day\"}],\\\n",
    "                   title=DEFAULTTITLE+\" (RIVM query)\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Sort days by sentiment score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "{k:v for k,v in sorted(sentimentPerDay.items(),key=lambda item:item[1][SENTIMENT])}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Examine tweet texts from day with extreme sentiment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gzip\n",
    "import json\n",
    "import numpy as np\n",
    "from library import getTweetText\n",
    "\n",
    "def removeNewlines(text):\n",
    "    return(re.sub(r\"\\n\",r\" \",text))\n",
    "\n",
    "HOURSPERDAY = 24\n",
    "DATE = \"20200901\"\n",
    "TWEETDIRECTORYRIVM = \"../data/rivm/\"\n",
    "\n",
    "tweetTexts = []\n",
    "for hour in range(0,HOURSPERDAY):\n",
    "    hour = str(hour).zfill(2)\n",
    "    inFile = gzip.open(TWEETDIRECTORYRIVM+DATE+\"-\"+hour+\".rivm.gz\")\n",
    "    for line in inFile:\n",
    "        jsonData = json.loads(line)\n",
    "        tweetText = removeNewlines(getTweetText(jsonData))\n",
    "        tweetTexts.append(tweetText)\n",
    "    inFile.close()\n",
    "    \n",
    "pd.DataFrame(tweetTexts)[0].value_counts()[0:20]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Process sentiment of topic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def readSentiment(fileName):\n",
    "    return(pd.read_csv(fileName,index_col=\"date\").T.to_dict())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "DATADIR = \"../data/sentiment/\"\n",
    "TOPIC = \"corona|covid|flattenthecurve|blijfthuis|rivm|mondkapje|huisarts|houvol\"\n",
    "EXTENSION = \".csv\"\n",
    "\n",
    "# sentimentPerHourTopic = readSentiment(DATADIR+TOPIC+EXTENSION)\n",
    "sentimentPerHourTopic = getSentiment()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sentimentPerDayTopic = makeSentimentPerDay(sentimentPerHourTopic)\n",
    "dummy = visualizeSentiment([{DATA:sentimentPerDay,LABEL:\"all\",\n",
    "                            HIGHLIGHT:highlight,HIGHLIGHTLABEL:\"press conference\"},\n",
    "                            {DATA:sentimentPerDayTopic,LABEL:\"topic\",\\\n",
    "                             HIGHLIGHT:highlight,HIGHLIGHTLABEL:\"press conference\"}],\\\n",
    "                           title=DEFAULTTITLE+\" (all Dutch tweets vs topic tweets)\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for selectedDates in \"202002 202003 202004 202005[01]\".split():\n",
    "    nbrOfTweetsAll = sum([sentimentPerDay[date][COUNT] for date in sentimentPerDay if re.search(selectedDates,date)])\n",
    "    nbrOfTweetsTopic = sum([sentimentPerDayTopic[date][COUNT] for date in sentimentPerDayTopic if re.search(selectedDates,date)])\n",
    "    print(selectedDates,round(nbrOfTweetsTopic/nbrOfTweetsAll,3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sentimentPerDay"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}