20190419.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# FASTTEXT EXPERIMENTS WITH LARGE DATA SET"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Read 4M words\n",
      "Number of words:  284034\n",
      "Number of labels: 16\n",
      "Progress: 100.0% words/sec/thread:  990928 lr:  0.000000 loss:  0.441493 ETA:   0h 0m\n"
     ]
    }
   ],
   "source": [
    "fasttext supervised -input link-articles-20190417-fasttext-train.txt -output model -epoch 100 -dim 100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  915  65.25  65.25  65.25 \n",
      "  129  44.85  57.36  50.34 __label__ACH\n",
      "   39  28.00  35.90  31.46 __label__COL\n",
      "   18   0.00   0.00   0.00 __label__ESS\n",
      "    9  63.64  77.78  70.00 __label__FIC\n",
      "   32  85.71  37.50  52.17 __label__ING\n",
      "   37  66.67  37.84  48.28 __label__INT\n",
      "   30  66.67  60.00  63.16 __label__LOS\n",
      "   22  78.57  50.00  61.11 __label__MED\n",
      "  345  80.65  85.80  83.15 __label__NIE\n",
      "   32  42.86  56.25  48.65 __label__OPI\n",
      "   14  14.29   7.14   9.52 __label__OVE\n",
      "   11  33.33  18.18  23.53 __label__POR\n",
      "   73  77.27  69.86  73.38 __label__REC\n",
      "   19  25.00  21.05  22.86 __label__REP\n",
      "   77  75.31  79.22  77.22 __label__SER\n",
      "   28  50.00  50.00  50.00 __label__VER\n"
     ]
    }
   ],
   "source": [
    "fasttext predict model.bin link-articles-20190417-fasttext-test.txt |\\\n",
    "   paste -d' ' - link-articles-20190417-fasttext-test.txt |\\\n",
    "   cut -d' ' -f1,2 | sed 's/\\(.*\\) \\(.*\\)/\\2 \\1/' | sed 's/^/x /'|\\\n",
    "   python ../../online-behaviour/machine-learning/eval.py"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Collapsing all but target classes\n",
    "\n",
    "Target genre classes: ACH COL INT NIE OPI REC REP VER\n",
    "\n",
    "Other genre classes: ESS FIC ING LOS MED OVE POR SER"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "python ../data-processing/collapse-labels.py < link-articles-20190417-fasttext-test.txt > \\\n",
    "                                               link-articles-20190417-fasttext-test-collapsed.txt\n",
    "python ../data-processing/collapse-labels.py < link-articles-20190417-fasttext-train.txt > \\\n",
    "                                               link-articles-20190417-fasttext-train-collapsed.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Read 4M words\n",
      "Number of words:  284034\n",
      "Number of labels: 16\n",
      "Progress: 100.0% words/sec/thread:  994183 lr:  0.000000 loss:  0.439971 ETA:   0h 0m\n"
     ]
    }
   ],
   "source": [
    "fasttext supervised -input link-articles-20190417-fasttext-train-collapsed.txt -output model-collapsed -epoch 100 -dim 100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  915  64.70  64.70  64.70 \n",
      "  129  44.17  55.81  49.32 __label__ACH\n",
      "   39  26.53  33.33  29.55 __label__COL\n",
      "   18   0.00   0.00   0.00 __label__ESS\n",
      "    9  70.00  77.78  73.68 __label__FIC\n",
      "   32  85.71  37.50  52.17 __label__ING\n",
      "   37  65.00  35.14  45.61 __label__INT\n",
      "   30  66.67  60.00  63.16 __label__LOS\n",
      "   22  84.62  50.00  62.86 __label__MED\n",
      "  345  80.22  85.80  82.91 __label__NIE\n",
      "   32  42.86  56.25  48.65 __label__OPI\n",
      "   14  14.29   7.14   9.52 __label__OVE\n",
      "   11  28.57  18.18  22.22 __label__POR\n",
      "   73  75.76  68.49  71.94 __label__REC\n",
      "   19  26.67  21.05  23.53 __label__REP\n",
      "   77  73.49  79.22  76.25 __label__SER\n",
      "   28  48.28  50.00  49.12 __label__VER\n"
     ]
    }
   ],
   "source": [
    "fasttext predict model-collapsed.bin link-articles-20190417-fasttext-test-collapsed.txt |\\\n",
    "   paste -d' ' - link-articles-20190417-fasttext-test-collapsed.txt |\\\n",
    "   cut -d' ' -f1,2 | sed 's/\\(.*\\) \\(.*\\)/\\2 \\1/' | sed 's/^/x /'|\\\n",
    "   python ../../online-behaviour/machine-learning/eval.py"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Including additional metadata fields\n",
    "\n",
    "Interesting metadata fields: Aard nieuws, Zelfclassificatie, Directe quotes, Onderwerp, Soort Auteur"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Read 4M words\n",
      "Number of words:  284074\n",
      "Number of labels: 18\n",
      "Progress: 100.0% words/sec/thread: 1081173 lr:  0.000000 loss:  0.461047 ETA:   0h 0m\n"
     ]
    }
   ],
   "source": [
    "fasttext supervised -input link-articles-20190419-fasttext-metadata-train.txt -output model -epoch 100 -dim 100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  915  64.81  64.81  64.81 \n",
      "  129  44.24  56.59  49.66 __label__ACH\n",
      "   39  32.56  35.90  34.15 __label__COL\n",
      "   18   0.00   0.00   0.00 __label__ESS\n",
      "    9  80.00  88.89  84.21 __label__FIC\n",
      "   12  25.00  33.33  28.57 __label__HOO\n",
      "   32  85.71  37.50  52.17 __label__ING\n",
      "   37  56.00  37.84  45.16 __label__INT\n",
      "   30  75.00  60.00  66.67 __label__LOS\n",
      "    7   0.00   0.00   0.00 __label__MED\n",
      "   15  90.00  60.00  72.00 __label__MOP\n",
      "  345  80.65  86.96  83.68 __label__NIE\n",
      "   20  31.58  30.00  30.77 __label__OPI\n",
      "   14  14.29   7.14   9.52 __label__OVE\n",
      "   11  20.00   9.09  12.50 __label__PRO\n",
      "   73  76.47  71.23  73.76 __label__REC\n",
      "   19  27.27  31.58  29.27 __label__REP\n",
      "   77  69.32  79.22  73.94 __label__SER\n",
      "   28  53.85  50.00  51.85 __label__VER\n"
     ]
    }
   ],
   "source": [
    "fasttext predict model.bin link-articles-20190419-fasttext-metadata-test.txt |\\\n",
    "   paste -d' ' - link-articles-20190419-fasttext-metadata-test.txt |\\\n",
    "   cut -d' ' -f1,2 | sed 's/\\(.*\\) \\(.*\\)/\\2 \\1/' | sed 's/^/x /'|\\\n",
    "   python ../../online-behaviour/machine-learning/eval.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "python ../data-processing/collapse-labels.py -l 9 < link-articles-20190419-fasttext-metadata-test.txt > \\\n",
    "                                                    link-articles-20190419-fasttext-metadata-test-collapsed.txt\n",
    "python ../data-processing/collapse-labels.py -l 9 < link-articles-20190419-fasttext-metadata-train.txt > \\\n",
    "                                                    link-articles-20190419-fasttext-metadata-train-collapsed.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Read 4M words\n",
      "Number of words:  284074\n",
      "Number of labels: 9\n",
      "Progress: 100.0% words/sec/thread: 1013899 lr:  0.000000 loss:  0.314105 ETA:   0h 0m\n"
     ]
    }
   ],
   "source": [
    "fasttext supervised -input link-articles-20190419-fasttext-metadata-train-collapsed.txt -output model -epoch 100 -dim 100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  915  66.99  66.99  66.99 \n",
      "  129  45.61  60.47  52.00 __label__ACH\n",
      "   39  34.48  25.64  29.41 __label__COL\n",
      "   37  72.22  35.14  47.27 __label__INT\n",
      "  345  80.44  84.64  82.49 __label__NIE\n",
      "   32  40.00  37.50  38.71 __label__OPI\n",
      "   73  76.92  68.49  72.46 __label__REC\n",
      "   19  35.00  36.84  35.90 __label__REP\n",
      "  213  70.98  64.32  67.49 __label__SER\n",
      "   28  53.85  50.00  51.85 __label__VER\n"
     ]
    }
   ],
   "source": [
    "fasttext predict model.bin link-articles-20190419-fasttext-metadata-test-collapsed.txt |\\\n",
    "   paste -d' ' - link-articles-20190419-fasttext-metadata-test-collapsed.txt |\\\n",
    "   cut -d' ' -f1,2 | sed 's/\\(.*\\) \\(.*\\)/\\2 \\1/' | sed 's/^/dummy /'|\\\n",
    "   python ../../online-behaviour/machine-learning/eval.py"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Binary labels\n",
    "\n",
    "Opinionated genres: ACH + COL + OPI + REC \n",
    "\n",
    "Objective genres: INT + NIE + REP + VER\n",
    "\n",
    "Other genre: SER"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MED = MOP + MED\n",
      "OPI = HOO + OPI\n",
      "POR = PRO + POR\n",
      "SER = ESS + FIC + ING + LOS + MED + OVE + POR + SER\n",
      "NIE = INT + REP + VER + NIE\n",
      "COL = ACH + OPI + REC + COL\n"
     ]
    }
   ],
   "source": [
    "python ../data-processing/collapse-labels.py -s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "python ../data-processing/collapse-labels.py -l 3 < link-articles-20190417-fasttext-test.txt > \\\n",
    "                                                    link-articles-20190417-fasttext-test-collapsed.txt\n",
    "python ../data-processing/collapse-labels.py -l 3 < link-articles-20190417-fasttext-train.txt > \\\n",
    "                                                    link-articles-20190417-fasttext-train-collapsed.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Read 4M words\n",
      "Number of words:  284034\n",
      "Number of labels: 3\n",
      "Progress: 100.0% words/sec/thread: 1146170 lr:  0.000000 loss:  0.140549 ETA:   0h 0m\n"
     ]
    }
   ],
   "source": [
    "fasttext supervised -input link-articles-20190417-fasttext-train-collapsed.txt -output model-collapsed -epoch 100 -dim 100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  915  70.49  70.49  70.49 \n",
      "  273  62.09  69.60  65.63 __label__COL\n",
      "  429  76.96  75.52  76.24 __label__NIE\n",
      "  213  69.68  61.50  65.34 __label__SER\n"
     ]
    }
   ],
   "source": [
    "fasttext predict model-collapsed.bin link-articles-20190417-fasttext-test-collapsed.txt |\\\n",
    "   paste -d' ' - link-articles-20190417-fasttext-test-collapsed.txt |\\\n",
    "   cut -d' ' -f1,2 | sed 's/\\(.*\\) \\(.*\\)/\\2 \\1/' | sed 's/^/dummy /'|\\\n",
    "   python ../../online-behaviour/machine-learning/eval.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH=2 DIM=10: N\t915\tP@1\t0.561\tR@1\t0.561\t\n",
      "EPOCH=4 DIM=10: N\t915\tP@1\t0.662\tR@1\t0.662\t\n",
      "EPOCH=6 DIM=10: N\t915\tP@1\t0.707\tR@1\t0.707\t\n",
      "EPOCH=8 DIM=10: N\t915\tP@1\t0.726\tR@1\t0.726\t\n",
      "EPOCH=10 DIM=10: N\t915\tP@1\t0.727\tR@1\t0.727\t\n",
      "EPOCH=12 DIM=10: N\t915\tP@1\t0.731\tR@1\t0.731\t\n",
      "EPOCH=14 DIM=10: N\t915\tP@1\t0.728\tR@1\t0.728\t\n"
     ]
    }
   ],
   "source": [
    "DIM=10\n",
    "for EPOCH in 2 4 6 8 10 12 14\n",
    "do\n",
    "    fasttext supervised -input link-articles-20190417-fasttext-train-collapsed.txt -output model-collapsed \\\n",
    "        -epoch $EPOCH -dim $DIM  >/dev/null 2>/dev/null        \n",
    "    echo -e \"EPOCH=$EPOCH DIM=$DIM: \\c\"\n",
    "    fasttext test model-collapsed.bin link-articles-20190417-fasttext-test-collapsed.txt | (tr '\\n' '\\t'; echo)\n",
    "done"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH=12 DIM=10: N\t915\tP@1\t0.729\tR@1\t0.729\t\n",
      "EPOCH=12 DIM=20: N\t915\tP@1\t0.728\tR@1\t0.728\t\n",
      "EPOCH=12 DIM=50: N\t915\tP@1\t0.723\tR@1\t0.723\t\n",
      "EPOCH=12 DIM=100: N\t915\tP@1\t0.726\tR@1\t0.726\t\n",
      "EPOCH=12 DIM=200: N\t915\tP@1\t0.728\tR@1\t0.728\t\n",
      "EPOCH=12 DIM=500: N\t915\tP@1\t0.727\tR@1\t0.727\t\n"
     ]
    }
   ],
   "source": [
    "EPOCH=12\n",
    "for DIM in 10 20 50 100 200 500\n",
    "do\n",
    "    fasttext supervised -input link-articles-20190417-fasttext-train-collapsed.txt -output model-collapsed \\\n",
    "      -epoch $EPOCH -dim $DIM >/dev/null 2>/dev/null\n",
    "    echo -e \"EPOCH=$EPOCH DIM=$DIM: \\c\"\n",
    "    fasttext test model-collapsed.bin link-articles-20190417-fasttext-test-collapsed.txt| (tr '\\n' '\\t'; echo)\n",
    "done"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## Collapsing to binary labels after processing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Read 4M words\n",
      "Number of words:  284034\n",
      "Number of labels: 16\n",
      "Progress: 100.0% words/sec/thread: 1134970 lr:  0.000000 loss:  0.223982 ETA:   0h 0m\n"
     ]
    }
   ],
   "source": [
    "DIM=100\n",
    "EPOCH=200\n",
    "fasttext supervised -input link-articles-20190417-fasttext-train.txt -output model -epoch $EPOCH -dim $DIM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "N\t915\n",
      "P@1\t0.639\n",
      "R@1\t0.639\n"
     ]
    }
   ],
   "source": [
    "fasttext test model.bin link-articles-20190417-fasttext-test.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  915  73.44  73.44  73.44 \n",
      "  273  65.49  68.13  66.79 __label__COL\n",
      "  429  78.05  82.05  80.00 __label__NIE\n",
      "  213  74.44  62.91  68.19 __label__SER\n"
     ]
    }
   ],
   "source": [
    "L=3\n",
    "fasttext predict model.bin link-articles-20190417-fasttext-test.txt |\\\n",
    "    python ../data-processing/collapse-labels.py -l $L |\\\n",
    "    paste -d' ' - link-articles-20190417-fasttext-test.txt |\\\n",
    "    cut -d' ' -f1,2 | sed 's/\\(.*\\) \\(.*\\)/\\2 \\1/' |\\\n",
    "    python ../data-processing/collapse-labels.py -l $L | sed 's/^/dummy /'|\\\n",
    "    python ../../online-behaviour/machine-learning/eval.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    COL NIE SER\n",
      "COL 186  58  29\n",
      "NIE  60 352  17\n",
      "SER  38  41 134\n"
     ]
    }
   ],
   "source": [
    "L=3\n",
    "fasttext predict model.bin link-articles-20190417-fasttext-test.txt |\\\n",
    "    python ../data-processing/collapse-labels.py -l $L |\\\n",
    "    paste -d' ' - link-articles-20190417-fasttext-test.txt |\\\n",
    "    cut -d' ' -f1,2 | sed 's/\\(.*\\) \\(.*\\)/\\2 \\1/' |\\\n",
    "    python ../data-processing/collapse-labels.py -l $L |\\\n",
    "    sort | uniq -c |\\\n",
    "    python ../data-processing/data2matrix.py"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## Error analysis on development set with 9 genre labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "L=9\n",
    "grep -e DATE=02/../1965 -e DATE=11/../1985 ../data/link-articles-20190417-fasttext-train.txt |\\\n",
    "   cut -d' ' -f1,3,7- | \\\n",
    "   python ../data-processing/collapse-labels.py -l $L > link-articles-20190417-fasttext-dev-test.txt\n",
    "grep -v -e DATE=02/../1965 -e DATE=11/../1985 ../data/link-articles-20190417-fasttext-train.txt |\\\n",
    "   cut -d' ' -f1,3,7- | \\\n",
    "   python ../data-processing/collapse-labels.py -l $L > link-articles-20190417-fasttext-dev-train.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH=10 DIM=10: N\t738\tP@1\t0.585\tR@1\t0.585\t\n",
      "EPOCH=20 DIM=10: N\t738\tP@1\t0.618\tR@1\t0.618\t\n",
      "EPOCH=50 DIM=10: N\t738\tP@1\t0.692\tR@1\t0.692\t\n",
      "EPOCH=100 DIM=10: N\t738\tP@1\t0.694\tR@1\t0.694\t\n",
      "EPOCH=200 DIM=10: N\t738\tP@1\t0.694\tR@1\t0.694\t\n",
      "EPOCH=500 DIM=10: N\t738\tP@1\t0.695\tR@1\t0.695\t\n"
     ]
    }
   ],
   "source": [
    "DIM=10\n",
    "for EPOCH in 10 20 50 100 200 500\n",
    "do\n",
    "    fasttext supervised -input link-articles-20190417-fasttext-dev-train.txt -output model \\\n",
    "        -epoch $EPOCH -dim $DIM  >/dev/null 2>/dev/null        \n",
    "    echo -e \"EPOCH=$EPOCH DIM=$DIM: \\c\"\n",
    "    fasttext test model.bin link-articles-20190417-fasttext-dev-test.txt | (tr '\\n' '\\t'; echo)\n",
    "done"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH=50 DIM=10: N\t738\tP@1\t0.691\tR@1\t0.691\t\n",
      "EPOCH=50 DIM=20: N\t738\tP@1\t0.687\tR@1\t0.687\t\n",
      "EPOCH=50 DIM=50: N\t738\tP@1\t0.683\tR@1\t0.683\t\n",
      "EPOCH=50 DIM=100: N\t738\tP@1\t0.686\tR@1\t0.686\t\n",
      "EPOCH=50 DIM=200: N\t738\tP@1\t0.686\tR@1\t0.686\t\n"
     ]
    }
   ],
   "source": [
    "EPOCH=50\n",
    "for DIM in 10 20 50 100 200\n",
    "do\n",
    "    fasttext supervised -input link-articles-20190417-fasttext-dev-train.txt -output model \\\n",
    "        -epoch $EPOCH -dim $DIM  >/dev/null 2>/dev/null        \n",
    "    echo -e \"EPOCH=$EPOCH DIM=$DIM: \\c\"\n",
    "    fasttext test model.bin link-articles-20190417-fasttext-dev-test.txt | (tr '\\n' '\\t'; echo)\n",
    "done"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Source pretrained word vectors: https://fasttext.cc/docs/en/pretrained-vectors.html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH=5 DIM=300: N\t738\tP@1\t0.696\tR@1\t0.696\t\n",
      "EPOCH=10 DIM=300: N\t738\tP@1\t0.722\tR@1\t0.722\t\n",
      "EPOCH=20 DIM=300: N\t738\tP@1\t0.721\tR@1\t0.721\t\n",
      "EPOCH=50 DIM=300: N\t738\tP@1\t0.699\tR@1\t0.699\t\n"
     ]
    }
   ],
   "source": [
    "PRETRAINEDVECTORS=wiki.nl.vec\n",
    "DIM=300\n",
    "for EPOCH in 5 10 20 50\n",
    "do\n",
    "   fasttext supervised -input link-articles-20190417-fasttext-dev-train.txt -output model \\\n",
    "      -pretrainedVectors $PRETRAINEDVECTORS -epoch $EPOCH -dim $DIM 2>/dev/null >/dev/null\n",
    "   echo -e \"EPOCH=$EPOCH DIM=$DIM: \\c\"\n",
    "   fasttext test model.bin link-articles-20190417-fasttext-dev-test.txt | (tr '\\n' '\\t'; echo)\n",
    "done"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Read 3M words\n",
      "Number of words:  267769\n",
      "Number of labels: 9\n",
      "Progress: 100.0% words/sec/thread:  630491 lr:  0.000000 loss:  0.841228 ETA:   0h 0m\n"
     ]
    }
   ],
   "source": [
    "PRETRAINEDVECTORS=wiki.nl.vec\n",
    "DIM=300\n",
    "EPOCH=10\n",
    "fasttext supervised -input link-articles-20190417-fasttext-dev-train.txt -output model \\\n",
    "      -pretrainedVectors $PRETRAINEDVECTORS -epoch $EPOCH -dim $DIM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "function testGenres {\n",
    "    MODEL=$1\n",
    "    TESTFILE=$2\n",
    "    L=$3\n",
    "    fasttext predict $MODEL $TESTFILE |\\\n",
    "        if [ \"$L\" != \"\" ]; then python ../data-processing/collapse-labels.py -l $L\n",
    "        else cat; fi |\\\n",
    "        paste -d' ' - $TESTFILE |\\\n",
    "        cut -d' ' -f1,2 | sed 's/\\(.*\\) \\(.*\\)/\\2 \\1/' |\\\n",
    "        if [ \"$L\" != \"\" ]; then python ../data-processing/collapse-labels.py -l $L\n",
    "        else cat; fi |\\\n",
    "        sed 's/^/dummy /'|\\\n",
    "        python ../../online-behaviour/machine-learning/eval.py\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  738  71.27  71.27  71.27 \n",
      "  101  51.22  41.58  45.90 __label__ACH\n",
      "   14  37.50  21.43  27.27 __label__COL\n",
      "   14 100.00  14.29  25.00 __label__INT\n",
      "  282  77.22  86.52  81.61 __label__NIE\n",
      "   12  30.00  25.00  27.27 __label__OPI\n",
      "   34  79.41  79.41  79.41 __label__REC\n",
      "    9   0.00   0.00   0.00 __label__REP\n",
      "  173  76.19  73.99  75.07 __label__SER\n",
      "   99  65.25  77.78  70.97 __label__VER\n"
     ]
    }
   ],
   "source": [
    "testGenres model.bin link-articles-20190417-fasttext-dev-test.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "function testMatrix {\n",
    "    MODEL=$1\n",
    "    TESTFILE=$2\n",
    "    L=$3\n",
    "    fasttext predict $MODEL $TESTFILE |\\\n",
    "        if [ \"$L\" != \"\" ]; then python ../data-processing/collapse-labels.py -l $L\n",
    "        else cat; fi|\\\n",
    "        paste -d' ' - $TESTFILE |\\\n",
    "        cut -d' ' -f1,2 | sed 's/^\\([^ ]*\\) \\([^ ]*\\)/\\2 \\1/' |\\\n",
    "        if [ \"$L\" != \"\" ]; then python ../data-processing/collapse-labels.py -l $L\n",
    "        else cat; fi |\\\n",
    "        sort | uniq -c |\\\n",
    "        python ../data-processing/data2matrix.py\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    ACH COL INT NIE OPI REC REP SER VER\n",
      "ACH  42   .   .  37   2   3   .   3  14\n",
      "COL   5   3   .   .   1   1   .   3   1\n",
      "INT   5   4   2   1   .   .   .   2   .\n",
      "NIE   3   .   . 244   1   .   .  25   9\n",
      "OPI   8   .   .   .   3   .   .   1   .\n",
      "REC   3   .   .   .   .  27   .   3   1\n",
      "REP   6   .   .   2   .   .   .   1   .\n",
      "SER   5   1   .  17   3   3   . 128  16\n",
      "VER   5   .   .  15   .   .   .   2  77\n"
     ]
    }
   ],
   "source": [
    "testMatrix model.bin link-articles-20190417-fasttext-dev-test.txt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Ideas for better performances\n",
    "\n",
    "1. add number of words per article as feature\n",
    "2. add relative page number as feature\n",
    "3. build model for one year\n",
    "4. build model for one newspaper\n",
    "5. combine 3 and 4\n",
    "6. add weekday as feature"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "L=9\n",
    "grep -e DATE=02/../1965 -e DATE=11/../1985 ../data/link-articles-20190417-fasttext-train.txt |\\\n",
    "   python ../data-processing/fixPageFeature.py |\\\n",
    "   python ../data-processing/addWeekDay.py |\\\n",
    "   cut -d' ' -f1-4,6,10- |\\\n",
    "   python ../data-processing/collapse-labels.py -l $L > link-articles-20190417-fasttext-dev-test.txt\n",
    "grep -v -e DATE=02/../1965 -e DATE=11/../1985 ../data/link-articles-20190417-fasttext-train.txt |\\\n",
    "   python ../data-processing/fixPageFeature.py |\\\n",
    "   python ../data-processing/addWeekDay.py |\\\n",
    "   cut -d' ' -f1-4,6,10- |\\\n",
    "   python ../data-processing/collapse-labels.py -l $L > link-articles-20190417-fasttext-dev-train.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH=5 DIM=300: N\t738\tP@1\t0.714\tR@1\t0.714\t\n",
      "EPOCH=10 DIM=300: N\t738\tP@1\t0.729\tR@1\t0.729\t\n",
      "EPOCH=20 DIM=300: N\t738\tP@1\t0.728\tR@1\t0.728\t\n",
      "EPOCH=50 DIM=300: N\t738\tP@1\t0.714\tR@1\t0.714\t\n"
     ]
    }
   ],
   "source": [
    "PRETRAINEDVECTORS=wiki.nl.vec\n",
    "DIM=300\n",
    "for EPOCH in 5 10 20 50\n",
    "do\n",
    "   fasttext supervised -input link-articles-20190417-fasttext-dev-train.txt -output model \\\n",
    "      -pretrainedVectors $PRETRAINEDVECTORS -epoch $EPOCH -dim $DIM 2>/dev/null >/dev/null\n",
    "   echo -e \"EPOCH=$EPOCH DIM=$DIM: \\c\"\n",
    "   fasttext test model.bin link-articles-20190417-fasttext-dev-test.txt | (tr '\\n' '\\t'; echo)\n",
    "done"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Read 3M words\n",
      "Number of words:  267780\n",
      "Number of labels: 9\n",
      "Progress: 100.0% words/sec/thread:  623525 lr:  0.000000 loss:  0.835678 ETA:   0h 0m\n"
     ]
    }
   ],
   "source": [
    "PRETRAINEDVECTORS=wiki.nl.vec\n",
    "DIM=300\n",
    "EPOCH=10\n",
    "fasttext supervised -input link-articles-20190417-fasttext-dev-train.txt -output model \\\n",
    "      -pretrainedVectors $PRETRAINEDVECTORS -epoch $EPOCH -dim $DIM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  738  72.36  72.36  72.36 \n",
      "  101  51.69  45.54  48.42 __label__ACH\n",
      "   14  37.50  21.43  27.27 __label__COL\n",
      "   14 100.00  14.29  25.00 __label__INT\n",
      "  282  79.17  87.59  83.16 __label__NIE\n",
      "   12  30.00  25.00  27.27 __label__OPI\n",
      "   34  80.00  82.35  81.16 __label__REC\n",
      "    9   0.00   0.00   0.00 __label__REP\n",
      "  173  78.88  73.41  76.05 __label__SER\n",
      "   99  64.46  78.79  70.91 __label__VER\n"
     ]
    }
   ],
   "source": [
    "testGenres model.bin link-articles-20190417-fasttext-dev-test.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    ACH COL INT NIE OPI REC REP SER VER\n",
      "ACH  46   .   .  33   2   3   .   3  14\n",
      "COL   5   3   .   .   1   1   .   3   1\n",
      "INT   6   4   2   .   .   .   .   2   .\n",
      "NIE   5   .   . 247   1   .   .  20   9\n",
      "OPI   8   .   .   .   3   .   .   1   .\n",
      "REC   2   .   .   .   .  28   .   3   1\n",
      "REP   7   .   .   1   .   .   .   1   .\n",
      "SER   4   1   .  17   3   3   . 127  18\n",
      "VER   6   .   .  14   .   .   .   1  78\n"
     ]
    }
   ],
   "source": [
    "testMatrix model.bin link-articles-20190417-fasttext-dev-test.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  738  78.59  78.59  78.59 \n",
      "  161  71.83  63.35  67.33 __label__COL\n",
      "  404  80.69  86.88  83.67 __label__NIE\n",
      "  173  78.88  73.41  76.05 __label__SER\n"
     ]
    }
   ],
   "source": [
    "testGenres model.bin link-articles-20190417-fasttext-dev-test.txt 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    COL NIE SER\n",
      "COL 102  49  10\n",
      "NIE  29 351  24\n",
      "SER  11  35 127\n"
     ]
    }
   ],
   "source": [
    "testMatrix model.bin link-articles-20190417-fasttext-dev-test.txt 3"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "### 11cv experiments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-rw-r--r-- 1 erikt erikt 1435546 May 10 13:58 link-articles-20190417-fasttext-train-11.txt\n",
      "-rw-r--r-- 1 erikt erikt 1550469 May 10 13:58 link-articles-20190417-fasttext-train-12.txt\n",
      "-rw-r--r-- 1 erikt erikt 1669063 May 10 13:58 link-articles-20190417-fasttext-train-13.txt\n",
      "-rw-r--r-- 1 erikt erikt 1288958 May 10 13:58 link-articles-20190417-fasttext-train-14.txt\n",
      "-rw-r--r-- 1 erikt erikt 1606238 May 10 13:58 link-articles-20190417-fasttext-train-15.txt\n",
      "-rw-r--r-- 1 erikt erikt 3169801 May 10 13:58 link-articles-20190417-fasttext-train-16.txt\n",
      "-rw-r--r-- 1 erikt erikt 2515222 May 10 13:58 link-articles-20190417-fasttext-train-17.txt\n",
      "-rw-r--r-- 1 erikt erikt 2139758 May 10 13:58 link-articles-20190417-fasttext-train-18.txt\n",
      "-rw-r--r-- 1 erikt erikt 2307274 May 10 13:58 link-articles-20190417-fasttext-train-19.txt\n",
      "-rw-r--r-- 1 erikt erikt 2092902 May 10 13:58 link-articles-20190417-fasttext-train-20.txt\n",
      "-rw-r--r-- 1 erikt erikt 2545828 May 10 13:58 link-articles-20190417-fasttext-train-21.txt\n"
     ]
    }
   ],
   "source": [
    "ls -l link-articles-20190417-fasttext-train-??.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "PRETRAINEDVECTORS=wiki.nl.vec\n",
    "DIM=300\n",
    "EPOCH=10\n",
    "\n",
    "function makeTrain {\n",
    "    BASEFILENAME=$1\n",
    "    N=$2\n",
    "    NEWSPAPER=$3\n",
    "    if [ -z \"$N\" ]\n",
    "    then\n",
    "       echo \"usage: makeTrain BASEFILENAME N [NEWSPAPER]\" >&2\n",
    "       exit 1\n",
    "    fi\n",
    "    echo -e \"\\c\" > TRAIN\n",
    "    for I in 11 12 13 14 15 16 17 18 19 20 21\n",
    "    do\n",
    "        if [ $I != $N ]\n",
    "        then\n",
    "            cat $BASEFILENAME-$I.txt |\\\n",
    "                if [ \"$NEWSPAPER\" == \"\" ]\n",
    "                then\n",
    "                    cat\n",
    "                else\n",
    "                    grep NEWSPAPER=$NEWSPAPER\n",
    "                fi >> TRAIN\n",
    "        fi\n",
    "    done\n",
    "}\n",
    "\n",
    "function makeTest {\n",
    "    BASEFILENAME=$1\n",
    "    N=$2\n",
    "    NEWSPAPER=$3 \n",
    "    if [ -z \"$N\" ]\n",
    "    then\n",
    "       echo \"usage: makeTest BASEFILENAME N [NEWSPAPER]\" >&2\n",
    "       exit 1\n",
    "    fi\n",
    "    cat $BASEFILENAME-$N.txt |\\\n",
    "        if [ \"$NEWSPAPER\" == \"\" ]\n",
    "        then\n",
    "            cat\n",
    "        else\n",
    "            grep NEWSPAPER=$NEWSPAPER\n",
    "        fi > TEST\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "11:   705  71.63  71.63  71.63 \n",
      "12:   722  74.93  74.93  74.93 \n",
      "13:   800  74.25  74.25  74.25 \n",
      "14:   626  69.33  69.33  69.33 \n",
      "15:   724  67.82  67.82  67.82 \n",
      "16:   961  65.35  65.35  65.35 \n",
      "17:   805  68.20  68.20  68.20 \n",
      "18:   718  72.98  72.98  72.98 \n",
      "19:   778  63.50  63.50  63.50 \n",
      "20:   703  64.86  64.86  64.86 \n",
      "21:   789  67.05  67.05  67.05 \n"
     ]
    }
   ],
   "source": [
    "BASEFILENAME=link-articles-20190417-fasttext-train\n",
    "\n",
    "for N in 11 12 13 14 15 16 17 18 19 20 21\n",
    "do\n",
    "    if [ ! -f model$EPOCH-$N.vec ]\n",
    "    then\n",
    "        makeTrain $BASEFILENAME $N\n",
    "        fasttext supervised -input TRAIN -output model$EPOCH-$N -pretrainedVectors $PRETRAINEDVECTORS \\\n",
    "            -epoch $EPOCH -dim $DIM 2>/dev/null >/dev/null\n",
    "    fi\n",
    "    makeTest $BASEFILENAME $N\n",
    "    echo -e \"$N: \\c\"\n",
    "    testGenres model$EPOCH-$N.bin TEST | head -1\n",
    "done"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "11:   705  73.19  73.19  73.19 \n",
      "12:   722  75.90  75.90  75.90 \n",
      "13:   800  76.12  76.12  76.12 \n",
      "14:   626  70.29  70.29  70.29 \n",
      "15:   724  68.92  68.92  68.92 \n",
      "16:   961  67.53  67.53  67.53 \n",
      "17:   805  68.82  68.82  68.82 \n",
      "18:   718  73.96  73.96  73.96 \n",
      "19:   778  65.17  65.17  65.17 \n",
      "20:   703  66.15  66.15  66.15 \n",
      "21:   789  68.31  68.31  68.31 \n"
     ]
    }
   ],
   "source": [
    "BASEFILENAME=link-articles-20190417-fasttext-train\n",
    "NBROFGENRES=9\n",
    "for N in 11 12 13 14 15 16 17 18 19 20 21\n",
    "do\n",
    "    makeTest $BASEFILENAME $N\n",
    "    echo -e \"$N: \\c\"\n",
    "    testGenres model$N.bin TEST $NBROFGENRES | head -1\n",
    "done"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "11:   705  81.56  81.56  81.56 \n",
      "12:   722  82.13  82.13  82.13 \n",
      "13:   800  82.00  82.00  82.00 \n",
      "14:   626  75.72  75.72  75.72 \n",
      "15:   724  75.55  75.55  75.55 \n",
      "16:   961  75.55  75.55  75.55 \n",
      "17:   805  75.40  75.40  75.40 \n",
      "18:   718  79.25  79.25  79.25 \n",
      "19:   778  71.47  71.47  71.47 \n",
      "20:   703  71.41  71.41  71.41 \n",
      "21:   789  74.65  74.65  74.65 \n"
     ]
    }
   ],
   "source": [
    "NBROFGENRES=3\n",
    "for N in 11 12 13 14 15 16 17 18 19 20 21\n",
    "do\n",
    "    makeTest $N\n",
    "    echo -e \"$N: \\c\"\n",
    "    testGenres model$N.bin TEST $NBROFGENRES | head -1\n",
    "done"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "11:   705  72.20  72.20  72.20 \n",
      "12:   722  74.93  74.93  74.93 \n",
      "13:   800  73.88  73.88  73.88 \n",
      "14:   626  71.25  71.25  71.25 \n",
      "15:   724  70.03  70.03  70.03 \n",
      "16:   961  67.12  67.12  67.12 \n",
      "17:   805  65.59  65.59  65.59 \n",
      "18:   718  71.17  71.17  71.17 \n",
      "19:   778  68.51  68.51  68.51 \n",
      "20:   703  65.29  65.29  65.29 \n",
      "21:   789  69.33  69.33  69.33 \n"
     ]
    }
   ],
   "source": [
    "EPOCH=50\n",
    "\n",
    "for N in 11 12 13 14 15 16 17 18 19 20 21\n",
    "do\n",
    "    if [ ! -f model$N-$EPOCH.vec ]\n",
    "    then\n",
    "        makeTrain $N\n",
    "        fasttext supervised -input TRAIN -output model$N-$EPOCH -pretrainedVectors $PRETRAINEDVECTORS \\\n",
    "            -epoch $EPOCH -dim $DIM 2>/dev/null >/dev/null\n",
    "    fi\n",
    "    makeTest $N\n",
    "    echo -e \"$N: \\c\"\n",
    "    testGenres model$N-$EPOCH.bin TEST | head -1\n",
    "done"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "11:   705  81.28  81.28  81.28 \n",
      "12:   722  81.99  81.99  81.99 \n",
      "13:   800  83.00  83.00  83.00 \n",
      "14:   626  77.32  77.32  77.32 \n",
      "15:   724  77.62  77.62  77.62 \n",
      "16:   961  77.94  77.94  77.94 \n",
      "17:   805  76.15  76.15  76.15 \n",
      "18:   718  78.97  78.97  78.97 \n",
      "19:   778  73.91  73.91  73.91 \n",
      "20:   703  71.69  71.69  71.69 \n",
      "21:   789  76.30  76.30  76.30 \n"
     ]
    }
   ],
   "source": [
    "NBROFGENRES=3\n",
    "for N in 11 12 13 14 15 16 17 18 19 20 21\n",
    "do\n",
    "    makeTest $N\n",
    "    echo -e \"$N: \\c\"\n",
    "    testGenres model$N-$EPOCH.bin TEST $NBROFGENRES | head -1\n",
    "done"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "function testGenres11cv {\n",
    "   BASEFILENAME=$1\n",
    "   MODEL=$2\n",
    "   L=$3\n",
    "   NEWSPAPER=$4\n",
    "   for N in 11 12 13 14 15 16 17 18 19 20 21\n",
    "   do\n",
    "      makeTest $BASEFILENAME $N $NEWSPAPER\n",
    "      fasttext predict ${MODEL}-$N-20.bin TEST |\\\n",
    "         if [ \"$L\" != \"\" ]; then python ../data-processing/collapse-labels.py -l $L\n",
    "         else cat; fi |\\\n",
    "         paste -d' ' - TEST\n",
    "   done |\\\n",
    "   cut -d' ' -f1,2 | sed 's/\\(.*\\) \\(.*\\)/\\2 \\1/' |\\\n",
    "      if [ \"$L\" != \"\" ]; then python ../data-processing/collapse-labels.py -l $L\n",
    "      else cat; fi |\\\n",
    "   sed 's/^/dummy /'|\\\n",
    "   python ../../online-behaviour/machine-learning/eval.py\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 8331  76.75  76.75  76.75 \n",
      " 1987  63.79  68.44  66.04 __label__COL\n",
      " 4430  78.69  84.94  81.70 __label__NIE\n",
      " 1914  89.70  66.41  76.31 __label__SER\n"
     ]
    }
   ],
   "source": [
    "testGenres11cv model10 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 8331  77.84  77.84  77.84 \n",
      " 1987  65.18  71.41  68.16 __label__COL\n",
      " 4430  80.10  84.40  82.19 __label__NIE\n",
      " 1914  89.30  69.33  78.06 __label__SER\n"
     ]
    }
   ],
   "source": [
    "testGenres11cv model20 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 8331  76.87  76.87  76.87 \n",
      " 1987  64.68  68.75  66.65 __label__COL\n",
      " 4430  79.76  82.98  81.34 __label__NIE\n",
      " 1914  84.60  71.16  77.30 __label__SER\n"
     ]
    }
   ],
   "source": [
    "testGenres11cv model50 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "05NRC_Handelsblad\n",
      "11:   232  70.26  70.26  70.26 \n",
      "12:   244  72.95  72.95  72.95 \n",
      "13:   259  72.20  72.20  72.20 \n",
      "14:   183  73.77  73.77  73.77 \n",
      "15:   258  68.22  68.22  68.22 \n",
      "16:   239  63.18  63.18  63.18 \n",
      "17:   267  61.05  61.05  61.05 \n",
      "18:   242  59.92  59.92  59.92 \n",
      "19:   267  62.92  62.92  62.92 \n",
      "20:   250  57.60  57.60  57.60 \n",
      "21:   246  57.32  57.32  57.32 \n",
      "06De_Telegraaf\n",
      "11:   283  73.85  73.85  73.85 \n",
      "12:   260  79.62  79.62  79.62 \n",
      "13:   304  78.95  78.95  78.95 \n",
      "14:   236  66.95  66.95  66.95 \n",
      "15:   279  65.95  65.95  65.95 \n",
      "16:   347  71.18  71.18  71.18 \n",
      "17:   267  70.04  70.04  70.04 \n",
      "18:   234  76.07  76.07  76.07 \n",
      "19:   263  66.54  66.54  66.54 \n",
      "20:   220  71.82  71.82  71.82 \n",
      "21:   279  72.40  72.40  72.40 \n",
      "08De_Volkskrant\n",
      "11:   190  69.47  69.47  69.47 \n",
      "12:   218  69.72  69.72  69.72 \n",
      "13:   237  70.04  70.04  70.04 \n",
      "14:   207  70.05  70.05  70.05 \n",
      "15:   187  66.84  66.84  66.84 \n",
      "16:   375  59.20  59.20  59.20 \n",
      "17:   271  69.74  69.74  69.74 \n",
      "18:   242  72.73  72.73  72.73 \n",
      "19:   248  64.52  64.52  64.52 \n",
      "20:   233  61.80  61.80  61.80 \n",
      "21:   264  65.91  65.91  65.91 \n"
     ]
    }
   ],
   "source": [
    "EPOCH=20\n",
    "\n",
    "for NEWSPAPER in 05NRC_Handelsblad 06De_Telegraaf 08De_Volkskrant\n",
    "do\n",
    "    echo $NEWSPAPER\n",
    "    for N in 11 12 13 14 15 16 17 18 19 20 21\n",
    "    do\n",
    "        if [ ! -f model$EPOCH-$NEWSPAPER-$N.vec ]\n",
    "        then\n",
    "            makeTrain $N $NEWSPAPER\n",
    "            fasttext supervised -input TRAIN -output model$EPOCH-$NEWSPAPER-$N -pretrainedVectors $PRETRAINEDVECTORS \\\n",
    "                -epoch $EPOCH -dim $DIM 2>/dev/null >/dev/null\n",
    "        fi\n",
    "        makeTest $N $NEWSPAPER\n",
    "        echo -e \"$N: \\c\"\n",
    "        testGenres model$EPOCH-$NEWSPAPER-$N.bin TEST | head -1\n",
    "    done\n",
    "done"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "05NRC_Handelsblad\n",
      " 2687  65.17  65.17  65.17 \n",
      "  440  45.45  52.27  48.63 __label__ACH\n",
      "   58  44.68  36.21  40.00 __label__COL\n",
      "   27   0.00   0.00   0.00 __label__ESS\n",
      "   35  83.33  85.71  84.51 __label__FIC\n",
      "   35 100.00   2.86   5.56 __label__ING\n",
      "   38  50.00   7.89  13.64 __label__INT\n",
      "  110  86.54  40.91  55.56 __label__LOS\n",
      "   44  78.95  34.09  47.62 __label__MED\n",
      " 1161  72.76  86.74  79.14 __label__NIE\n",
      "  104  38.04  33.65  35.71 __label__OPI\n",
      "   55  14.29   1.82   3.23 __label__OVE\n",
      "   22   0.00   0.00   0.00 __label__POR\n",
      "  124  59.28  79.84  68.04 __label__REC\n",
      "   56  40.00  10.71  16.90 __label__REP\n",
      "  274  79.44  83.21  81.28 __label__SER\n",
      "  104  44.12  28.85  34.88 __label__VER\n",
      " 2687  73.32  73.32  73.32 \n",
      "  726  62.19  69.56  65.67 __label__COL\n",
      " 1359  75.02  81.31  78.04 __label__NIE\n",
      "  602  89.55  59.80  71.71 __label__SER\n",
      " 2687  76.29  76.29  76.29 \n",
      "  726  65.02  72.73  68.66 __label__COL\n",
      " 1359  78.27  82.41  80.29 __label__NIE\n",
      "  602  90.54  66.78  76.86 __label__SER\n",
      "06De_Telegraaf\n",
      " 2972  72.17  72.17  72.17 \n",
      "  368  43.15  58.15  49.54 __label__ACH\n",
      "   72  56.67  47.22  51.52 __label__COL\n",
      "   19   0.00   0.00   0.00 __label__ESS\n",
      "   26  83.33  57.69  68.18 __label__FIC\n",
      "   21   0.00   0.00   0.00 __label__ING\n",
      "   51  35.71  29.41  32.26 __label__INT\n",
      "  173  92.20  75.14  82.80 __label__LOS\n",
      "   44  71.43  56.82  63.29 __label__MED\n",
      " 1471  81.16  88.72  84.77 __label__NIE\n",
      "   53  66.67  15.09  24.62 __label__OPI\n",
      "   42   0.00   0.00   0.00 __label__OVE\n",
      "   23   0.00   0.00   0.00 __label__POR\n",
      "   71  73.33  61.97  67.18 __label__REC\n",
      "   35   0.00   0.00   0.00 __label__REP\n",
      "  302  79.80  81.13  80.46 __label__SER\n",
      "  201  56.99  54.73  55.84 __label__VER\n",
      " 2972  79.64  79.64  79.64 \n",
      "  564  61.15  68.09  64.43 __label__COL\n",
      " 1758  83.23  87.26  85.20 __label__NIE\n",
      "  650  89.62  69.08  78.02 __label__SER\n",
      " 2972  81.12  81.12  81.12 \n",
      "  564  65.21  68.79  66.95 __label__COL\n",
      " 1758  84.39  88.28  86.29 __label__NIE\n",
      "  650  87.55  72.46  79.29 __label__SER\n",
      "08De_Volkskrant\n",
      " 2672  66.80  66.80  66.80 \n",
      "  473  47.39  59.62  52.81 __label__ACH\n",
      "   54  48.72  35.19  40.86 __label__COL\n",
      "   16   0.00   0.00   0.00 __label__ESS\n",
      "   36  64.10  69.44  66.67 __label__FIC\n",
      "   42  50.00   7.14  12.50 __label__ING\n",
      "   44  27.27   6.82  10.91 __label__INT\n",
      "  121  85.33  52.89  65.31 __label__LOS\n",
      "   34  96.15  73.53  83.33 __label__MED\n",
      " 1065  73.76  82.35  77.82 __label__NIE\n",
      "   58  38.89  12.07  18.42 __label__OPI\n",
      "   73  15.38   2.74   4.65 __label__OVE\n",
      "   20   0.00   0.00   0.00 __label__POR\n",
      "  112  59.86  78.57  67.95 __label__REC\n",
      "   27   0.00   0.00   0.00 __label__REP\n",
      "  320  87.54  92.19  89.80 __label__SER\n",
      "  177  53.67  53.67  53.67 __label__VER\n",
      " 2672  74.10  74.10  74.10 \n",
      "  697  62.33  71.45  66.58 __label__COL\n",
      " 1313  75.53  79.21  77.32 __label__NIE\n",
      "  662  89.11  66.77  76.34 __label__SER\n",
      " 2672  75.75  75.75  75.75 \n",
      "  697  65.32  72.17  68.58 __label__COL\n",
      " 1313  76.32  81.26  78.72 __label__NIE\n",
      "  662  90.08  68.58  77.87 __label__SER\n"
     ]
    }
   ],
   "source": [
    "for NEWSPAPER in 05NRC_Handelsblad 06De_Telegraaf 08De_Volkskrant\n",
    "do\n",
    "    echo $NEWSPAPER\n",
    "    testGenres11cv model20-$NEWSPAPER 16 $NEWSPAPER\n",
    "    testGenres11cv model20-$NEWSPAPER 3 $NEWSPAPER\n",
    "    testGenres11cv model20 3 $NEWSPAPER\n",
    "done"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 8331  77.84  77.84  77.84 \n",
      " 1987  65.18  71.41  68.16 __label__COL\n",
      " 4430  80.10  84.40  82.19 __label__NIE\n",
      " 1914  89.30  69.33  78.06 __label__SER\n"
     ]
    }
   ],
   "source": [
    "testGenres11cv model20 3"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "### Error analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "function confusion {\n",
    "   BASEFILENAME=$1\n",
    "   MODEL=$2\n",
    "   L=$3\n",
    "   NEWSPAPER=$4\n",
    "   for N in 11 12 13 14 15 16 17 18 19 20 21\n",
    "   do\n",
    "      makeTest $BASEFILENAME $N $NEWSPAPER\n",
    "      fasttext predict ${MODEL}-$N-20.bin TEST |\\\n",
    "         if [ \"$L\" != \"\" ]; then python ../data-processing/collapse-labels.py -l $L\n",
    "         else cat; fi |\\\n",
    "      paste -d' ' - TEST\n",
    "   done |\\\n",
    "   cut -d' ' -f1,2 | sed 's/\\(.*\\) \\(.*\\)/\\2 \\1/' |\\\n",
    "      if [ \"$L\" != \"\" ]; then python ../data-processing/collapse-labels.py -l $L\n",
    "      else cat; fi |\\\n",
    "   sed 's/^/dummy /' |\\\n",
    "   python ../data-processing/confusion.py 2 3 | sed 's/^__label__/ /'\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NIE: 3213  104  263   69   25   12    2    9    . 3697\n",
      "SER:  322 1327   76   51   54   32   34   12    6 1914\n",
      "ACH:  412   14  745   49   21   16    6   16    2 1281\n",
      "VER:  128   20   79  243    5    5    1    1    .  482\n",
      "REC:   12    4   32    1  246    9    2    1    .  307\n",
      "OPI:   16    1   99    .   10   74   12    3    .  215\n",
      "COL:    7   11   36    5   11   19   81    8    6  184\n",
      "INT:    5    4   61    1    2    3    9   47    1  133\n",
      "REP:    2    1   75    7   11    .    9    8    5  118\n",
      "      4117 1486 1466  426  385  170  156  105   20\n"
     ]
    }
   ],
   "source": [
    "confusion model20 9"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Read 4M words\n",
      "Number of words:  284034\n",
      "Number of labels: 9\n",
      "Progress: 100.0% words/sec/thread:  619135 lr:  0.000000 loss:  0.661340 ETA:   0h 0m\n"
     ]
    }
   ],
   "source": [
    "# 20200129 for clin30 talk\n",
    "CELLTRAIN=link-articles-20190417-fasttext-train.txt\n",
    "CELLTEST=link-articles-20190417-fasttext-test.txt\n",
    "CELLMODEL=cellmodel\n",
    "CELLDIM=300\n",
    "CELLEPOCH=20\n",
    "CELLPRETRAINEDVECTORS=\"wiki.nl.vec\"\n",
    "\n",
    "python3 ../data-processing/collapse-labels.py -l 9 < $CELLTRAIN > $CELLTRAIN.9\n",
    "python3 ../data-processing/collapse-labels.py -l 9 < $CELLTEST > $CELLTEST.9\n",
    "\n",
    "fasttext supervised -input $CELLTRAIN.9 -output $CELLMODEL -dim $CELLDIM -epoch $CELLEPOCH \\\n",
    "                    -pretrainedVectors $CELLPRETRAINEDVECTORS\n",
    "fasttext predict $CELLMODEL.bin $CELLTEST.9 > $CELLMODEL.out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 8331  71.79  71.79  71.79 \n",
      " 1281  50.82  58.16  54.24 __label__ACH\n",
      "  184  51.92  44.02  47.65 __label__COL\n",
      "  133  44.76  35.34  39.50 __label__INT\n",
      " 3697  78.04  86.91  82.24 __label__NIE\n",
      "  215  43.53  34.42  38.44 __label__OPI\n",
      "  307  63.90  80.13  71.10 __label__REC\n",
      "  118  25.00   4.24   7.25 __label__REP\n",
      " 1914  89.30  69.33  78.06 __label__SER\n",
      "  482  57.04  50.41  53.52 __label__VER\n"
     ]
    }
   ],
   "source": [
    "testGenres11cv model20 9"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 8331  70.52  70.52  70.52 \n",
      " 1281  50.82  58.16  54.24 __label__ACH\n",
      "  184  51.92  44.02  47.65 __label__COL\n",
      "   62   0.00   0.00   0.00 __label__ESS\n",
      "   97  74.51  78.35  76.38 __label__FIC\n",
      "   98  62.50  10.20  17.54 __label__ING\n",
      "  133  44.76  35.34  39.50 __label__INT\n",
      "  404  85.67  66.58  74.93 __label__LOS\n",
      "  122  86.05  60.66  71.15 __label__MED\n",
      " 3697  78.04  86.91  82.24 __label__NIE\n",
      "  215  43.53  34.42  38.44 __label__OPI\n",
      "  170  32.00   4.71   8.21 __label__OVE\n",
      "   65  42.86   9.23  15.19 __label__POR\n",
      "  307  63.90  80.13  71.10 __label__REC\n",
      "  118  25.00   4.24   7.25 __label__REP\n",
      "  896  83.75  86.83  85.26 __label__SER\n",
      "  482  57.04  50.41  53.52 __label__VER\n"
     ]
    }
   ],
   "source": [
    "testGenres11cv model20"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Marking quoted text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "TRAIN=../data/link-articles-20190417-fasttext-train.txt\n",
    "TEST=../data/link-articles-20190417-fasttext-test.txt\n",
    "\n",
    "cut -d' ' -f1,7- $TEST | python3 ../data-processing/removeQuotes.py > link-articles-20190417-fasttext-test-rmq.txt\n",
    "grep DATE=0[23]/../1965 $TRAIN | cut -d' ' -f1,7- |\\\n",
    "   python3 ../data-processing/removeQuotes.py > link-articles-20190417-fasttext-train-rmq-11.txt\n",
    "grep DATE=0[45]/../1965 $TRAIN | cut -d' ' -f1,7- |\\\n",
    "   python3 ../data-processing/removeQuotes.py > link-articles-20190417-fasttext-train-rmq-12.txt\n",
    "grep DATE=0[67]/../1965 $TRAIN | cut -d' ' -f1,7- |\\\n",
    "   python3 ../data-processing/removeQuotes.py > link-articles-20190417-fasttext-train-rmq-13.txt\n",
    "grep DATE=0[89]/../1965 $TRAIN | cut -d' ' -f1,7- |\\\n",
    "   python3 ../data-processing/removeQuotes.py > link-articles-20190417-fasttext-train-rmq-14.txt\n",
    "grep DATE=1[01]/../1965 $TRAIN | cut -d' ' -f1,7- |\\\n",
    "   python3 ../data-processing/removeQuotes.py > link-articles-20190417-fasttext-train-rmq-15.txt\n",
    "grep -e DATE=12/../1965 -e DATE=01/../1985 $TRAIN | cut -d' ' -f1,7- |\\\n",
    "   python3 ../data-processing/removeQuotes.py > link-articles-20190417-fasttext-train-rmq-16.txt\n",
    "grep DATE=0[23]/../1985 $TRAIN | cut -d' ' -f1,7- |\\\n",
    "   python3 ../data-processing/removeQuotes.py > link-articles-20190417-fasttext-train-rmq-17.txt\n",
    "grep DATE=0[45]/../1985 $TRAIN | cut -d' ' -f1,7- |\\\n",
    "   python3 ../data-processing/removeQuotes.py > link-articles-20190417-fasttext-train-rmq-18.txt\n",
    "grep DATE=0[67]/../1985 $TRAIN | cut -d' ' -f1,7- |\\\n",
    "   python3 ../data-processing/removeQuotes.py > link-articles-20190417-fasttext-train-rmq-19.txt\n",
    "grep DATE=0[89]/../1985 $TRAIN | cut -d' ' -f1,7- |\\\n",
    "   python3 ../data-processing/removeQuotes.py > link-articles-20190417-fasttext-train-rmq-20.txt\n",
    "grep DATE=1[01]/../1985 $TRAIN | cut -d' ' -f1,7- |\\\n",
    "   python3 ../data-processing/removeQuotes.py > link-articles-20190417-fasttext-train-rmq-21.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "11:   705  72.62  72.62  72.62 \n",
      "12:   722  75.76  75.76  75.76 \n",
      "13:   800  74.25  74.25  74.25 \n",
      "14:   626  70.61  70.61  70.61 \n",
      "15:   724  69.34  69.34  69.34 \n",
      "16:   961  67.22  67.22  67.22 \n",
      "17:   805  68.57  68.57  68.57 \n",
      "18:   718  72.14  72.14  72.14 \n",
      "19:   778  67.35  67.35  67.35 \n",
      "20:   703  64.01  64.01  64.01 \n",
      "21:   789  69.84  69.84  69.84 \n"
     ]
    }
   ],
   "source": [
    "EPOCH=20\n",
    "MODEL=model-rmq\n",
    "BASEFILENAME=link-articles-20190417-fasttext-train-rmq\n",
    "\n",
    "for N in 11 12 13 14 15 16 17 18 19 20 21\n",
    "do\n",
    "    if [ ! -f $MODEL-$N-$EPOCH.vec ]\n",
    "    then\n",
    "        makeTrain $BASEFILENAME $N \n",
    "        fasttext supervised -input TRAIN -output $MODEL-$N-$EPOCH -pretrainedVectors $PRETRAINEDVECTORS \\\n",
    "            -epoch $EPOCH -dim $DIM 2>/dev/null >/dev/null\n",
    "    fi\n",
    "    makeTest $BASEFILENAME $N\n",
    "    echo -e \"$N: \\c\"\n",
    "    testGenres $MODEL-$N-$EPOCH.bin TEST | head -1\n",
    "done"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 8331  70.08  70.08  70.08 \n",
      " 1281  50.10  58.16  53.83 __label__ACH\n",
      "  184  46.28  47.28  46.77 __label__COL\n",
      "   62   0.00   0.00   0.00 __label__ESS\n",
      "   97  77.17  73.20  75.13 __label__FIC\n",
      "   98  50.00  10.20  16.95 __label__ING\n",
      "  133  42.45  33.83  37.66 __label__INT\n",
      "  404  84.14  64.36  72.93 __label__LOS\n",
      "  122  83.52  62.30  71.36 __label__MED\n",
      " 3697  78.40  86.18  82.10 __label__NIE\n",
      "  215  46.55  37.67  41.65 __label__OPI\n",
      "  170  28.00   4.12   7.18 __label__OVE\n",
      "   65  50.00   7.69  13.33 __label__POR\n",
      "  307  63.57  80.13  70.89 __label__REC\n",
      "  118  10.00   1.69   2.90 __label__REP\n",
      "  896  83.82  86.72  85.24 __label__SER\n",
      "  482  55.68  49.79  52.57 __label__VER\n"
     ]
    }
   ],
   "source": [
    "BASEFILENAME=link-articles-20190417-fasttext-train-rmq\n",
    "\n",
    "testGenres11cv $BASEFILENAME model-rmq 16"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 8331  71.37  71.37  71.37 \n",
      " 1281  50.10  58.16  53.83 __label__ACH\n",
      "  184  46.28  47.28  46.77 __label__COL\n",
      "  133  42.45  33.83  37.66 __label__INT\n",
      " 3697  78.40  86.18  82.10 __label__NIE\n",
      "  215  46.55  37.67  41.65 __label__OPI\n",
      "  307  63.57  80.13  70.89 __label__REC\n",
      "  118  10.00   1.69   2.90 __label__REP\n",
      " 1914  89.15  68.65  77.57 __label__SER\n",
      "  482  55.68  49.79  52.57 __label__VER\n"
     ]
    }
   ],
   "source": [
    "BASEFILENAME=link-articles-20190417-fasttext-train-rmq\n",
    "\n",
    "testGenres11cv $BASEFILENAME model-rmq 9"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 8331  77.31  77.31  77.31 \n",
      " 1987  63.73  71.72  67.49 __label__COL\n",
      " 4430  80.11  83.57  81.80 __label__NIE\n",
      " 1914  89.15  68.65  77.57 __label__SER\n"
     ]
    }
   ],
   "source": [
    "BASEFILENAME=link-articles-20190417-fasttext-train-rmq\n",
    "\n",
    "testGenres11cv $BASEFILENAME model-rmq 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " NIE: 3186  278   69   72   25   29   11    2    8   12    2    .    2    1    .    . 3697\n",
      " ACH:  411  745    7   50    3   20   13    7    1   19    .    2    1    2    .    . 1281\n",
      " SER:   71    3  777    5   12    9    .    5    4    1    7    .    .    1    1    .  896\n",
      " VER:  118   92   13  240    1    5    5    2    2    1    .    1    1    .    1    .  482\n",
      " LOS:  114    2   10    2  260    3    1    2    .    .    1    1    .    8    .    .  404\n",
      " REC:    8   34    3    1    1  246    8    3    .    1    1    .    1    .    .    .  307\n",
      " OPI:   15   90    .    1    .   10   81   15    .    1    .    .    1    1    .    .  215\n",
      " COL:    5   34    2    8    .   12   20   87    .    4    1    4    3    4    .    .  184\n",
      " OVE:   74   16   26   36    .    8    1    1    7    .    .    1    .    .    .    .  170\n",
      " INT:    5   61    .    .    .    1    1   16    .   45    .    2    .    1    1    .  133\n",
      " MED:   15    1   17    1    3    3    .    3    2    .   76    .    .    1    .    .  122\n",
      " REP:    2   74    .    6    .   11    .   12    1   10    .    2    .    .    .    .  118\n",
      " ING:   19    8    2    .    .   11   27   17    .    .    2    1   10    1    .    .   98\n",
      " FIC:    1    2    .    .    3    .    .    9    .    5    .    5    1   71    .    .   97\n",
      " POR:   15   17    .    8    .   12    .    .    .    6    .    1    .    1    5    .   65\n",
      " ESS:    5   30    1    1    1    7    6    7    .    1    1    .    .    .    2    .   62\n",
      "      4064 1487  927  431  309  387  174  188   25  106   91   20   20   92   10    0\n"
     ]
    }
   ],
   "source": [
    "BASEFILENAME=link-articles-20190417-fasttext-train-rmq\n",
    "\n",
    "confusion $BASEFILENAME model-rmq 16"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " NIE: 3186  107  278   72   29   11    2   12    . 3697\n",
      " SER:  314 1314   79   53   53   35   44   13    9 1914\n",
      " ACH:  411   14  745   50   20   13    7   19    2 1281\n",
      " VER:  118   18   92  240    5    5    2    1    1  482\n",
      " REC:    8    6   34    1  246    8    3    1    .  307\n",
      " OPI:   15    2   90    1   10   81   15    1    .  215\n",
      " COL:    5   10   34    8   12   20   87    4    4  184\n",
      " INT:    5    2   61    .    1    1   16   45    2  133\n",
      " REP:    2    1   74    6   11    .   12   10    2  118\n",
      "      4064 1474 1487  431  387  174  188  106   20\n"
     ]
    }
   ],
   "source": [
    "BASEFILENAME=link-articles-20190417-fasttext-train-rmq\n",
    "\n",
    "confusion $BASEFILENAME model-rmq 9"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Bash",
   "language": "bash",
   "name": "bash"
  },
  "language_info": {
   "codemirror_mode": "shell",
   "file_extension": ".sh",
   "mimetype": "text/x-sh",
   "name": "bash"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}