notes_sLDA2.ipynb

{
 "metadata": {
  "name": "notes_sLDA2"
 }, 
 "nbformat": 2, 
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "markdown", 
     "source": [
      "# Running sLDA on another data set"
     ]
    }, 
    {
     "cell_type": "markdown", 
     "source": [
      "Let's run sLDA on another data set that we're familiar with and where we know there are understandable topics."
     ]
    }, 
    {
     "cell_type": "code", 
     "collapsed": true, 
     "input": [
      "from nltk import *", 
      "from nltk.corpus import PlaintextCorpusReader", 
      "from pandas import *"
     ], 
     "language": "python", 
     "outputs": [], 
     "prompt_number": 1
    }, 
    {
     "cell_type": "markdown", 
     "source": [
      "## Input data"
     ]
    }, 
    {
     "cell_type": "markdown", 
     "source": [
      "Top 5 topics in the congressional vote transcript training set."
     ]
    }, 
    {
     "cell_type": "code", 
     "collapsed": false, 
     "input": [
      "ls data_stage_three/training_set/ | sed 's/_.*//g' | uniq -c | sort -nr | sed 5q"
     ], 
     "language": "python", 
     "outputs": [
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        " 329 031", 
        " 296 088", 
        " 285 132", 
        " 158 621", 
        " 132 321"
       ]
      }
     ], 
     "prompt_number": 2
    }, 
    {
     "cell_type": "markdown", 
     "source": [
      "The top three, and their apparent topic (from reading a few):", 
      "", 
      "* 031: immigration?", 
      "", 
      "* 088: health coverage and budget?", 
      "", 
      "* 132: energy?"
     ]
    }, 
    {
     "cell_type": "markdown", 
     "source": [
      "### Master words list"
     ]
    }, 
    {
     "cell_type": "code", 
     "collapsed": false, 
     "input": [
      "%time training = PlaintextCorpusReader('data_stage_three/training_set/', '..._......_......._....txt')"
     ], 
     "language": "python", 
     "outputs": [
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "CPU times: user 0.04 s, sys: 0.02 s, total: 0.06 s", 
        "Wall time: 0.06 s"
       ]
      }
     ], 
     "prompt_number": 3
    }, 
    {
     "cell_type": "code", 
     "collapsed": false, 
     "input": [
      "%time all_words = DataFrame([(i,w2) for i,w2 in enumerate(FreqDist([w for w in training.words() if w.isalpha()]).keys())], columns=['wid','word'])", 
      "all_words = all_words.set_index('word')"
     ], 
     "language": "python", 
     "outputs": [
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "CPU times: user 5.39 s, sys: 0.23 s, total: 5.62 s", 
        "Wall time: 5.71 s"
       ]
      }
     ], 
     "prompt_number": 4
    }, 
    {
     "cell_type": "code", 
     "collapsed": false, 
     "input": [
      "all_words.head(2)"
     ], 
     "language": "python", 
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">", 
        "<table border=\"1\" class=\"dataframe\">", 
        "  <thead>", 
        "    <tr style=\"text-align: right;\">", 
        "      <th></th>", 
        "      <th>wid</th>", 
        "    </tr>", 
        "    <tr>", 
        "      <th>word</th>", 
        "      <th></th>", 
        "    </tr>", 
        "  </thead>", 
        "  <tbody>", 
        "    <tr>", 
        "      <th>the</th>", 
        "      <td> 0</td>", 
        "    </tr>", 
        "    <tr>", 
        "      <th>to</th>", 
        "      <td> 1</td>", 
        "    </tr>", 
        "  </tbody>", 
        "</table>", 
        "</div>"
       ], 
       "output_type": "pyout", 
       "prompt_number": 5, 
       "text": [
        "      wid", 
        "word     ", 
        "the     0", 
        "to      1"
       ]
      }
     ], 
     "prompt_number": 5
    }, 
    {
     "cell_type": "code", 
     "collapsed": false, 
     "input": [
      "%time a2 = DataFrame([(i,w2) for i,w2 in enumerate(FreqDist([w for w in training.words() if w.isalpha()]).keys())], columns=['wid','word'])", 
      "a2 = a2.set_index('wid')", 
      "a2.head(2)"
     ], 
     "language": "python", 
     "outputs": [
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "CPU times: user 3.55 s, sys: 0.18 s, total: 3.73 s", 
        "Wall time: 3.69 s"
       ]
      }, 
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">", 
        "<table border=\"1\" class=\"dataframe\">", 
        "  <thead>", 
        "    <tr style=\"text-align: right;\">", 
        "      <th></th>", 
        "      <th>word</th>", 
        "    </tr>", 
        "    <tr>", 
        "      <th>wid</th>", 
        "      <th></th>", 
        "    </tr>", 
        "  </thead>", 
        "  <tbody>", 
        "    <tr>", 
        "      <th>0</th>", 
        "      <td> the</td>", 
        "    </tr>", 
        "    <tr>", 
        "      <th>1</th>", 
        "      <td>  to</td>", 
        "    </tr>", 
        "  </tbody>", 
        "</table>", 
        "</div>"
       ], 
       "output_type": "pyout", 
       "prompt_number": 77, 
       "text": [
        "    word", 
        "wid     ", 
        "0    the", 
        "1     to"
       ]
      }
     ], 
     "prompt_number": 77
    }, 
    {
     "cell_type": "markdown", 
     "source": [
      "###3 bills"
     ]
    }, 
    {
     "cell_type": "markdown", 
     "source": [
      "The statements for 3 bills from the convote training set."
     ]
    }, 
    {
     "cell_type": "code", 
     "collapsed": true, 
     "input": [
      "bills = [31,88,132]"
     ], 
     "language": "python", 
     "outputs": [], 
     "prompt_number": 6
    }, 
    {
     "cell_type": "code", 
     "collapsed": true, 
     "input": [
      "TERMS_MAX = 40000", 
      "TERMS_FIRST = 0", 
      "TERMS_LAST = TERMS_FIRST + TERMS_MAX"
     ], 
     "language": "python", 
     "outputs": [], 
     "prompt_number": 148
    }, 
    {
     "cell_type": "code", 
     "collapsed": false, 
     "input": [
      "corpora = [(bill,PlaintextCorpusReader('data_stage_three/training_set/', '%03d_......_......._....txt'%bill)) for bill in bills]", 
      "c2 = [(bill,fileid,[w for w in c.words(fileid) if w.isalpha()]) for (bill,c) in corpora for fileid in c.fileids()]", 
      "c3 = DataFrame(c2,columns=['bill','fileid','words'])", 
      "c3 = c3.set_index('fileid')", 
      "%time c3['features'] = c3.words.apply(lambda words: [(w, all_words.wid.ix[w], cnt) for w,cnt in FreqDist(words).iteritems() if (all_words.wid.ix[w] >= TERMS_FIRST) & (all_words.wid.ix[w] <= TERMS_LAST)])", 
      "c3['feature_cnt'] = c3.features.apply(lambda feats: len(feats))", 
      "c3['word_cnt'] = c3.words.apply(lambda words: len(words))", 
      "c3.head(2)"
     ], 
     "language": "python", 
     "outputs": [
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "CPU times: user 7.61 s, sys: 0.11 s, total: 7.72 s", 
        "Wall time: 7.65 s"
       ]
      }, 
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">", 
        "<table border=\"1\" class=\"dataframe\">", 
        "  <thead>", 
        "    <tr style=\"text-align: right;\">", 
        "      <th></th>", 
        "      <th>bill</th>", 
        "      <th>words</th>", 
        "      <th>features</th>", 
        "      <th>feature_cnt</th>", 
        "      <th>word_cnt</th>", 
        "    </tr>", 
        "    <tr>", 
        "      <th>fileid</th>", 
        "      <th></th>", 
        "      <th></th>", 
        "      <th></th>", 
        "      <th></th>", 
        "      <th></th>", 
        "    </tr>", 
        "  </thead>", 
        "  <tbody>", 
        "    <tr>", 
        "      <th>031_400009_0182113_DON.txt</th>", 
        "      <td> 31</td>", 
        "      <td> [mr, chairman, i, rise, in, strong, opposition...</td>", 
        "      <td> [(of, 2, 9), (the, 0, 8), (this, 8, 7), (to, 1...</td>", 
        "      <td> 130</td>", 
        "      <td> 218</td>", 
        "    </tr>", 
        "    <tr>", 
        "      <th>031_400019_0182097_ROY.txt</th>", 
        "      <td> 31</td>", 
        "      <td> [mr, chairman, on, rollcall, no, i, was, unavo...</td>", 
        "      <td> [(i, 9, 3), (no, 61, 2), (been, 55, 1), (chair...</td>", 
        "      <td>  15</td>", 
        "      <td>  18</td>", 
        "    </tr>", 
        "  </tbody>", 
        "</table>", 
        "</div>"
       ], 
       "output_type": "pyout", 
       "prompt_number": 149, 
       "text": [
        "                            bill  \\", 
        "fileid                             ", 
        "031_400009_0182113_DON.txt    31   ", 
        "031_400019_0182097_ROY.txt    31   ", 
        "", 
        "                                                                        words  \\", 
        "fileid                                                                          ", 
        "031_400009_0182113_DON.txt  [mr, chairman, i, rise, in, strong, opposition...   ", 
        "031_400019_0182097_ROY.txt  [mr, chairman, on, rollcall, no, i, was, unavo...   ", 
        "", 
        "                                                                     features  \\", 
        "fileid                                                                          ", 
        "031_400009_0182113_DON.txt  [(of, 2, 9), (the, 0, 8), (this, 8, 7), (to, 1...   ", 
        "031_400019_0182097_ROY.txt  [(i, 9, 3), (no, 61, 2), (been, 55, 1), (chair...   ", 
        "", 
        "                            feature_cnt  word_cnt  ", 
        "fileid                                             ", 
        "031_400009_0182113_DON.txt          130       218  ", 
        "031_400019_0182097_ROY.txt           15        18  "
       ]
      }
     ], 
     "prompt_number": 149
    }, 
    {
     "cell_type": "code", 
     "collapsed": false, 
     "input": [
      "#c3.features[:30].values"
     ], 
     "language": "python", 
     "outputs": [], 
     "prompt_number": 150
    }, 
    {
     "cell_type": "markdown", 
     "source": [
      "***REDUCE TERMS: REMOVE INFREQUENT (<3?)***"
     ]
    }, 
    {
     "cell_type": "code", 
     "collapsed": true, 
     "input": [
      "MIN_FEATS = 5"
     ], 
     "language": "python", 
     "outputs": [], 
     "prompt_number": 151
    }, 
    {
     "cell_type": "code", 
     "collapsed": false, 
     "input": [
      "c3_1 = c3[c3.feature_cnt >= MIN_FEATS]", 
      "print len(c3),len(c3_1)"
     ], 
     "language": "python", 
     "outputs": [
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "910 882"
       ]
      }
     ], 
     "prompt_number": 152
    }, 
    {
     "cell_type": "code", 
     "collapsed": false, 
     "input": [
      "c4 = c3_1.reindex(np.random.permutation(c3_1.index))", 
      "#c4.head()", 
      "TRN = floor(.9*len(c4))", 
      "print TRN", 
      "trn = c4[:TRN]", 
      "dev = c4[TRN:]", 
      "print 'trn:',len(trn),'dev:',len(dev)"
     ], 
     "language": "python", 
     "outputs": [
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "793.0", 
        "trn: 793 dev: 89"
       ]
      }
     ], 
     "prompt_number": 153
    }, 
    {
     "cell_type": "markdown", 
     "source": [
      "Training files."
     ]
    }, 
    {
     "cell_type": "code", 
     "collapsed": true, 
     "input": [
      "train_data  = 'tmp.sLDA/est_in/train-data.dat'", 
      "train_label = 'tmp.sLDA/est_in/train-label.dat'"
     ], 
     "language": "python", 
     "outputs": [], 
     "prompt_number": 154
    }, 
    {
     "cell_type": "code", 
     "collapsed": false, 
     "input": [
      "!mkdir -p tmp.sLDA/est_in", 
      "!rm tmp.sLDA/est_in/*", 
      "!mkdir tmp.sLDA/est_out", 
      "!rm tmp.sLDA/est_out/*", 
      "with open(train_data,'w') as f:", 
      "    for i,feats in trn.features.iteritems():", 
      "        f.write(str(len(feats))+' ')", 
      "        for w,wid,cnt in feats:", 
      "            f.write(str(wid)+':'+str(cnt)+' ')", 
      "        f.write('\\n')", 
      "!head -n 2 $train_data"
     ], 
     "language": "python", 
     "outputs": [
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "mkdir: tmp.sLDA/est_out: File exists"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "9 45:1 29:1 15:1 140:1 9:1 18:1 160:1 1182:1 39:1 ", 
        "20 3:1 288:1 27:1 625:1 64:1 498:1 978:1 11:1 86:1 9:1 962:1 115:1 18:1 2:1 16:1 169:1 202:1 32:1 0:1 87:1 "
       ]
      }
     ], 
     "prompt_number": 155
    }, 
    {
     "cell_type": "code", 
     "collapsed": false, 
     "input": [
      "with open(train_label,'w') as f:", 
      "    for i,bill in trn.bill.iteritems():", 
      "        f.write(str(bills.index(bill))+'\\n')", 
      "!head -n 2 $train_label"
     ], 
     "language": "python", 
     "outputs": [
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "0", 
        "0"
       ]
      }
     ], 
     "prompt_number": 156
    }, 
    {
     "cell_type": "markdown", 
     "source": [
      "##Estimate parameters"
     ]
    }, 
    {
     "cell_type": "code", 
     "collapsed": true, 
     "input": [
      "settings    = 'tmp.sLDA/settings.txt'", 
      "output      = 'tmp.sLDA/est_out'"
     ], 
     "language": "python", 
     "outputs": [], 
     "prompt_number": 157
    }, 
    {
     "cell_type": "code", 
     "collapsed": false, 
     "input": [
      "initialize = 'random'", 
      "alpha = 1", 
      "k = 10", 
      "%time !slda est $train_data $train_label $settings $alpha $k $initialize $output"
     ], 
     "language": "python", 
     "outputs": [
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "", 
        "reading data from tmp.sLDA/est_in/train-data.dat", 
        "number of docs  : 793", 
        "number of terms : 17374", 
        "number of total words : 231543", 
        "", 
        "reading labels from tmp.sLDA/est_in/train-label.dat", 
        "number of classes : 3", 
        "", 
        "alpha is fixed ...", 
        "var max iter 20", 
        "var convergence 1.00E-03", 
        "em max iter 50", 
        "em convergence 1.00E-04", 
        "L2 penalty 1.00E-02", 
        "number of topics is 10", 
        "models will be saved in tmp.sLDA/est_out", 
        "initializing ...", 
        "**** em iteration 1 ****", 
        "**** e-step ****", 
        "document 0"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 100"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 200"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 300"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 400"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 500"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 600"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 700"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "likelihood: -2273018.9757934641", 
        "**** m-step ****", 
        "maximizing ...", 
        "step: 00 -> f: -871.063355"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 10 -> f: -868.601744"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 20 -> f: -867.531881"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 30 -> f: -867.525983"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 40 -> f: -867.525960"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "final f: -867.525960", 
        "**** em iteration 2 ****", 
        "**** e-step ****", 
        "document 0"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 100"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 200"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 300"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 400"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 500"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 600"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 700"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "likelihood: -1507622.7574473228", 
        "**** m-step ****", 
        "maximizing ...", 
        "step: 00 -> f: -859.970812"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 10 -> f: -854.506871"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 20 -> f: -854.336169"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 30 -> f: -854.334142"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 40 -> f: -854.334098"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "final f: -854.334097", 
        "**** em iteration 3 ****", 
        "**** e-step ****", 
        "document 0"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 100"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 200"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 300"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 400"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 500"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 600"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 700"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "likelihood: -1506224.0304529613", 
        "**** m-step ****", 
        "maximizing ...", 
        "step: 00 -> f: -827.407123"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 10 -> f: -820.270130"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 20 -> f: -819.216405"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 30 -> f: -819.177267"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 40 -> f: -819.175020"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "final f: -819.175002", 
        "**** em iteration 4 ****", 
        "**** e-step ****", 
        "document 0"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 100"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 200"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 300"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 400"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 500"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 600"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 700"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "likelihood: -1503177.2280319906", 
        "**** m-step ****", 
        "maximizing ...", 
        "step: 00 -> f: -774.040484"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 10 -> f: -765.629444"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 20 -> f: -764.654787"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 30 -> f: -764.534706"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 40 -> f: -764.519993"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "final f: -764.515402", 
        "**** em iteration 5 ****", 
        "**** e-step ****", 
        "document 0"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 100"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 200"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 300"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 400"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 500"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 600"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 700"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "likelihood: -1496432.2455900824", 
        "**** m-step ****", 
        "maximizing ...", 
        "step: 00 -> f: -690.165396"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 10 -> f: -678.956458"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 20 -> f: -676.690818"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 30 -> f: -676.084842"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 40 -> f: -676.081189"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "final f: -676.080967", 
        "**** em iteration 6 ****", 
        "**** e-step ****", 
        "document 0"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 100"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 200"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 300"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 400"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 500"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 600"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 700"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "likelihood: -1484527.1586060175", 
        "**** m-step ****", 
        "maximizing ...", 
        "step: 00 -> f: -585.187781"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 10 -> f: -578.848347"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 20 -> f: -576.977521"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 30 -> f: -576.434763"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 40 -> f: -576.403724"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "final f: -576.403301", 
        "**** em iteration 7 ****", 
        "**** e-step ****", 
        "document 0"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 100"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 200"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 300"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 400"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 500"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 600"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 700"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "likelihood: -1470018.0485302592", 
        "**** m-step ****", 
        "maximizing ...", 
        "step: 00 -> f: -528.030605"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 10 -> f: -524.786528"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 20 -> f: -524.086589"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 30 -> f: -524.037074"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 40 -> f: -524.003260"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "final f: -524.001927", 
        "**** em iteration 8 ****", 
        "**** e-step ****", 
        "document 0"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 100"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 200"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 300"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 400"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 500"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 600"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 700"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "likelihood: -1458216.1186789728", 
        "**** m-step ****", 
        "maximizing ...", 
        "step: 00 -> f: -511.175604"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 10 -> f: -509.345758"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 20 -> f: -508.879548"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 30 -> f: -508.806410"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 40 -> f: -508.784084"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "final f: -508.781850", 
        "**** em iteration 9 ****", 
        "**** e-step ****", 
        "document 0"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 100"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 200"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 300"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 400"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 500"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 600"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 700"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "likelihood: -1450538.1537685285", 
        "**** m-step ****", 
        "maximizing ...", 
        "step: 00 -> f: -505.746341"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 10 -> f: -504.366826"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 20 -> f: -504.187017"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 30 -> f: -504.063782"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 40 -> f: -504.030548"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "final f: -504.021507", 
        "**** em iteration 10 ****", 
        "**** e-step ****", 
        "document 0"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 100"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 200"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 300"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 400"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 500"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 600"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 700"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "likelihood: -1445601.6388559607", 
        "**** m-step ****", 
        "maximizing ..."
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 00 -> f: -503.353872"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 10 -> f: -502.345979"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 20 -> f: -502.199590"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 30 -> f: -502.158743"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 40 -> f: -502.141151"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "final f: -502.136964"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "**** em iteration 11 ****", 
        "**** e-step ****", 
        "document 0"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 100"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 200"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 300"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 400"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 500"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 600"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 700"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "likelihood: -1442270.4857979137", 
        "**** m-step ****", 
        "maximizing ..."
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 00 -> f: -502.214123"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 10 -> f: -501.487902"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 20 -> f: -501.355637"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 30 -> f: -501.334010"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 40 -> f: -501.328434"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "final f: -501.327282", 
        "**** em iteration 12 ****", 
        "**** e-step ****", 
        "document 0"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 100"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 200"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 300"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 400"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 500"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 600"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 700"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "likelihood: -1439961.0331355070", 
        "**** m-step ****", 
        "maximizing ...", 
        "step: 00 -> f: -500.871161"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 10 -> f: -500.377558"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 20 -> f: -500.205836"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 30 -> f: -500.188982"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 40 -> f: -500.176727"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "final f: -500.176298", 
        "**** em iteration 13 ****", 
        "**** e-step ****", 
        "document 0"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 100"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 200"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 300"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 400"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 500"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 600"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 700"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "likelihood: -1438325.4951675546", 
        "**** m-step ****", 
        "maximizing ...", 
        "step: 00 -> f: -498.604930"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 10 -> f: -498.125556"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 20 -> f: -498.058935"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 30 -> f: -498.043802"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 40 -> f: -498.038836"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "final f: -498.038541", 
        "**** em iteration 14 ****", 
        "**** e-step ****", 
        "document 0"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 100"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 200"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 300"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 400"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 500"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 600"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 700"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "likelihood: -1437126.3775377101", 
        "**** m-step ****", 
        "maximizing ...", 
        "step: 00 -> f: -495.554293"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 10 -> f: -495.102633"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 20 -> f: -495.068874"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 30 -> f: -495.056036"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 40 -> f: -495.053066"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "final f: -495.052638", 
        "**** em iteration 15 ****", 
        "**** e-step ****", 
        "document 0"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 100"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 200"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 300"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 400"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 500"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 600"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 700"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "likelihood: -1436214.1813086483", 
        "**** m-step ****", 
        "maximizing ...", 
        "step: 00 -> f: -491.641844"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 10 -> f: -491.217140"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 20 -> f: -491.177840"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 30 -> f: -491.168029"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 40 -> f: -491.166853"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "final f: -491.166294", 
        "**** em iteration 16 ****", 
        "**** e-step ****", 
        "document 0"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 100"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 200"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 300"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 400"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 500"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 600"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 700"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "likelihood: -1435493.6057805149", 
        "**** m-step ****", 
        "maximizing ...", 
        "step: 00 -> f: -487.018865"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 10 -> f: -486.611454"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 20 -> f: -486.574883"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 30 -> f: -486.562225"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 40 -> f: -486.558958"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "final f: -486.558042", 
        "**** em iteration 17 ****", 
        "**** e-step ****", 
        "document 0"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 100"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 200"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 300"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 400"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 500"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 600"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 700"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "likelihood: -1434914.0744736495", 
        "**** m-step ****", 
        "maximizing ...", 
        "step: 00 -> f: -481.992844"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 10 -> f: -481.595727"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 20 -> f: -481.571284"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 30 -> f: -481.557523"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 40 -> f: -481.554656"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "final f: -481.553639", 
        "**** em iteration 18 ****", 
        "**** e-step ****", 
        "document 0"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 100"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 200"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 300"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 400"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 500"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 600"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 700"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "likelihood: -1434440.6323457030", 
        "**** m-step ****", 
        "maximizing ..."
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 00 -> f: -476.690243"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 10 -> f: -476.288063"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 20 -> f: -476.268006"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 30 -> f: -476.258416"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 40 -> f: -476.257488"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "final f: -476.257072", 
        "**** em iteration 19 ****", 
        "**** e-step ****", 
        "document 0"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 100"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 200"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 300"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 400"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 500"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 600"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 700"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "likelihood: -1434056.4527275616", 
        "**** m-step ****", 
        "maximizing ..."
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 00 -> f: -470.637504"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 10 -> f: -470.167267"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 20 -> f: -470.145413"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 30 -> f: -470.139263"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 40 -> f: -470.138102"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "final f: -470.137426", 
        "**** em iteration 20 ****", 
        "**** e-step ****", 
        "document 0"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 100"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 200"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 300"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 400"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 500"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 600"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 700"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "likelihood: -1433740.5811901409", 
        "**** m-step ****", 
        "maximizing ...", 
        "step: 00 -> f: -464.568480"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 10 -> f: -464.095860"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 20 -> f: -464.057420"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 30 -> f: -464.054099"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 40 -> f: -464.050578"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "final f: -464.049916"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "**** em iteration 21 ****", 
        "**** e-step ****", 
        "document 0"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 100"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 200"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 300"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 400"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 500"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 600"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 700"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "likelihood: -1433451.3193553581", 
        "**** m-step ****", 
        "maximizing ...", 
        "step: 00 -> f: -458.641850"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 10 -> f: -458.206876"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 20 -> f: -458.158178"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 30 -> f: -458.153015"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 40 -> f: -458.152380"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "final f: -458.152179", 
        "**** em iteration 22 ****", 
        "**** e-step ****", 
        "document 0"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 100"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 200"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 300"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 400"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 500"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 600"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 700"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "likelihood: -1433194.2472689082", 
        "**** m-step ****"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "maximizing ...", 
        "step: 00 -> f: -453.431100"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 10 -> f: -452.979708"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 20 -> f: -452.953011"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 30 -> f: -452.948631"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 40 -> f: -452.948135"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "final f: -452.948108", 
        "**** em iteration 23 ****", 
        "**** e-step ****", 
        "document 0"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 100"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 200"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 300"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 400"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 500"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 600"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 700"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "likelihood: -1432966.4384903910", 
        "**** m-step ****", 
        "maximizing ..."
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 00 -> f: -449.024744"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 10 -> f: -448.616416"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 20 -> f: -448.600823"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 30 -> f: -448.597598"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 40 -> f: -448.596944"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "final f: -448.596834", 
        "**** em iteration 24 ****", 
        "**** e-step ****", 
        "document 0"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 100"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 200"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 300"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 400"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 500"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 600"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 700"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "likelihood: -1432759.2302151031", 
        "**** m-step ****", 
        "maximizing ...", 
        "step: 00 -> f: -445.704302"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 10 -> f: -445.337620"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 20 -> f: -445.323203"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 30 -> f: -445.321317"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 40 -> f: -445.321075"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "final f: -445.321007", 
        "**** em iteration 25 ****", 
        "**** e-step ****", 
        "document 0"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 100"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 200"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 300"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 400"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 500"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 600"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 700"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "likelihood: -1432570.8502718741", 
        "**** m-step ****", 
        "maximizing ...", 
        "step: 00 -> f: -443.629310"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 10 -> f: -443.344092"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 20 -> f: -443.333541"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 30 -> f: -443.330816"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 40 -> f: -443.330578"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "final f: -443.330542", 
        "**** em iteration 26 ****", 
        "**** e-step ****", 
        "document 0"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 100"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 200"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 300"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 400"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 500"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 600"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 700"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "likelihood: -1432388.4008468443", 
        "**** m-step ****", 
        "maximizing ...", 
        "step: 00 -> f: -443.340495"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 10 -> f: -443.097521"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 20 -> f: -443.073137"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 30 -> f: -443.063729"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 40 -> f: -443.062097"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "final f: -443.061916", 
        "**** em iteration 27 ****", 
        "**** e-step ****", 
        "document 0"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 100"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 200"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 300"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 400"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 500"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 600"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 700"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "likelihood: -1432207.2556523597", 
        "**** m-step ****", 
        "maximizing ...", 
        "step: 00 -> f: -444.297384"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 10 -> f: -443.981588"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 20 -> f: -443.964111"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 30 -> f: -443.958322"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 40 -> f: -443.957262"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "final f: -443.957161", 
        "**** em iteration 28 ****", 
        "**** e-step ****", 
        "document 0"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 100"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 200"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 300"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 400"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 500"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 600"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 700"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "likelihood: -1432022.5071083638", 
        "**** m-step ****"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "maximizing ...", 
        "step: 00 -> f: -446.480905"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 10 -> f: -446.078530"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 20 -> f: -446.055872"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 30 -> f: -446.050880"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 40 -> f: -446.050036"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "final f: -446.049881", 
        "**** em iteration 29 ****", 
        "**** e-step ****", 
        "document 0"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 100"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 200"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 300"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 400"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 500"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 600"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 700"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "likelihood: -1431827.5519916436", 
        "**** m-step ****", 
        "maximizing ...", 
        "step: 00 -> f: -448.607618"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 10 -> f: -448.317263"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 20 -> f: -448.299712"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 30 -> f: -448.291113"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 40 -> f: -448.288701"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "final f: -448.288136", 
        "**** em iteration 30 ****", 
        "**** e-step ****", 
        "document 0"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 100"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 200"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 300"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 400"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 500"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 600"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 700"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "likelihood: -1431633.3486085155", 
        "**** m-step ****", 
        "maximizing ..."
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 00 -> f: -450.136120"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 10 -> f: -450.010351"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 20 -> f: -450.002918"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 30 -> f: -449.997852"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 40 -> f: -449.995649"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "final f: -449.995342"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "**** em iteration 31 ****", 
        "**** e-step ****", 
        "document 0"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 100"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 200"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 300"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 400"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 500"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 600"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 700"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "likelihood: -1431452.1081838186", 
        "**** m-step ****", 
        "maximizing ...", 
        "step: 00 -> f: -450.935030"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 10 -> f: -450.893140"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 20 -> f: -450.884537"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 30 -> f: -450.881124"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 40 -> f: -450.880514"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "final f: -450.880414", 
        "**** em iteration 32 ****", 
        "**** e-step ****", 
        "document 0"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 100"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 200"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 300"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 400"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 500"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 600"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 700"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "likelihood: -1431287.4559640414", 
        "**** m-step ****", 
        "maximizing ...", 
        "step: 00 -> f: -451.374011"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 10 -> f: -451.351425"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 20 -> f: -451.347874"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 30 -> f: -451.346329"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 40 -> f: -451.345997"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "final f: -451.345966", 
        "**** em iteration 33 ****", 
        "**** e-step ****", 
        "document 0"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 100"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 200"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 300"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 400"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 500"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 600"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "document 700"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "likelihood: -1431147.9625147516", 
        "**** m-step ****"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "maximizing ...", 
        "step: 00 -> f: -451.585826"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 10 -> f: -451.572477"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 20 -> f: -451.569642"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 30 -> f: -451.569214"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "step: 40 -> f: -451.569128"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "final f: -451.569121"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "final e step document 0"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "final e step document 100"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "final e step document 200"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "final e step document 300"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "final e step document 400"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "final e step document 500"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "final e step document 600"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "final e step document 700"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "CPU times: user 1.15 s, sys: 0.19 s, total: 1.35 s"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "", 
        "Wall time: 247.93 s"
       ]
      }
     ], 
     "prompt_number": 158
    }, 
    {
     "cell_type": "code", 
     "collapsed": false, 
     "input": [
      "!ls $output"
     ], 
     "language": "python", 
     "outputs": [
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "010.gamma            010.model.text       020.model            030.gamma            030.model.text       final.model          likelihood.dat", 
        "010.model            020.gamma            020.model.text       030.model            final.gamma          final.model.text     word-assignments.dat"
       ]
      }
     ], 
     "prompt_number": 159
    }, 
    {
     "cell_type": "code", 
     "collapsed": false, 
     "input": [
      "!head $output/final.model.text | sed 5q"
     ], 
     "language": "python", 
     "outputs": [
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "alpha: 1.000000", 
        "number of topics: 10", 
        "size of vocab: 17374", 
        "number of classes: 3", 
        "betas: ", 
        "head: stdout: Broken pipe"
       ]
      }
     ], 
     "prompt_number": 160
    }, 
    {
     "cell_type": "code", 
     "collapsed": false, 
     "input": [
      "!head $output/final.gamma | sed 2q"
     ], 
     "language": "python", 
     "outputs": [
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "1.1529369676 1.1882527548 3.6781425728 1.0807487865 6.0910374573 1.1884162226 1.1328636779 1.1443032493 1.0479372862 1.2953610251", 
        "3.5379605549 2.5961729348 10.3539656577 1.7239186669 1.5741759399 1.4225806426 1.6717538381 2.0925558447 1.5885717443 3.4383441760"
       ]
      }
     ], 
     "prompt_number": 161
    }, 
    {
     "cell_type": "code", 
     "collapsed": false, 
     "input": [
      "!head $output/likelihood.dat"
     ], 
     "language": "python", 
     "outputs": [
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "-2273018.9757934641\t  inf", 
        "-1507622.7574473228\t3.36731e-01", 
        "-1506224.0304529613\t9.27770e-04", 
        "-1503177.2280319906\t2.02281e-03", 
        "-1496432.2455900824\t4.48715e-03", 
        "-1484527.1586060175\t7.95565e-03", 
        "-1470018.0485302592\t9.77356e-03", 
        "-1458216.1186789728\t8.02843e-03", 
        "-1450538.1537685285\t5.26531e-03", 
        "-1445601.6388559607\t3.40323e-03"
       ]
      }
     ], 
     "prompt_number": 162
    }, 
    {
     "cell_type": "code", 
     "collapsed": false, 
     "input": [
      "!head $output/word-assignments.dat | sed 2q"
     ], 
     "language": "python", 
     "outputs": [
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "009 0045:02 0029:04 0015:04 0140:04 0009:04 0018:02 0160:04 1182:04 0039:02", 
        "020 0003:02 0288:02 0027:02 0625:07 0064:02 0498:02 0978:00 0011:09 0086:02 0009:02 0962:09 0115:00 0018:02 0002:02 0016:02 0169:01 0202:02 0032:02 0000:02 0087:02"
       ]
      }
     ], 
     "prompt_number": 163
    }, 
    {
     "cell_type": "code", 
     "collapsed": false, 
     "input": [
      "topics = []", 
      "with open('%s/final.model.text'%output,'r') as f:", 
      "    i = 0", 
      "    while (True):", 
      "        l = f.readline().strip()", 
      "        if l == '': break", 
      "        l = l.strip()", 
      "        if l == 'betas:': ", 
      "            print 'betas'", 
      "            while(True):", 
      "                l2= f.readline().strip()", 
      "                if l2 == 'etas:': break", 
      "                w = [(wid,a2.word.ix[wid],float(n)) for wid,n in enumerate(l2.split(' '))]", 
      "                w.sort(key=lambda tup: tup[2])", 
      "                topics.append(w[:20])", 
      "                i += 1", 
      "print i", 
      "print len(topics)"
     ], 
     "language": "python", 
     "outputs": [
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "betas", 
        "10"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "", 
        "10"
       ]
      }
     ], 
     "prompt_number": 164
    }, 
    {
     "cell_type": "code", 
     "collapsed": false, 
     "input": [
      "for i,t in enumerate(topics):", 
      "    print i", 
      "    for wid,word,score in t:", 
      "        print word,", 
      "    print ''", 
      "    for wid,word,score in t:", 
      "        print score,", 
      "    print ''"
     ], 
     "language": "python", 
     "outputs": [
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "0", 
        "superpower civilization mainstream migrant undergraduate quest zealots trumpeted ofthe fanatics boldly supremacy arena advocating collegesand bribed tractor saturn freeway loomis ", 
        "-184.922201 -183.073103 -182.293072 -182.148448 -182.062952 -182.01931 -181.260308 -181.013779 -180.745359 -179.131928 -178.920901 -178.762549 -178.577636 -178.567951 -177.921184 -174.906718 -174.170002 -173.870947 -173.611074 -173.536064 ", 
        "1", 
        "superpower quest trumpeted undergraduate zealots mainstream migrant ofthe boldly fanatics arena collegesand civilization advocating supremacy windsor uncompleted eyesore thankfully odor ", 
        "-188.130632 -186.427877 -186.352047 -186.293984 -185.858012 -185.755212 -184.844308 -184.732441 -183.798961 -183.663708 -183.439594 -182.748469 -182.640814 -181.523922 -181.394873 -167.619009 -166.900405 -166.73448 -166.319061 -166.201698 ", 
        "2", 
        "collegesand zealots ofthe quest undergraduate fanatics civilization mainstream superpower migrant trumpeted arena boldly advocating supremacy hbcus spearheaded blooming multipurpose rails ", 
        "-189.39802 -189.325407 -187.465976 -187.170114 -187.162753 -186.937757 -186.550033 -186.289419 -186.175873 -185.864668 -185.778645 -185.773506 -185.363949 -183.776458 -182.248516 -167.148279 -161.714033 -161.196162 -161.090885 -161.013619 ", 
        "3", 
        "ofthe mainstream undergraduate migrant quest zealots trumpeted superpower fanatics arena supremacy collegesand civilization boldly advocating eleven prescribe verifying merrell followers ", 
        "-191.765193 -191.362936 -190.107796 -189.032611 -188.78288 -187.475495 -187.334786 -187.042426 -186.750704 -184.91807 -184.884109 -184.545872 -184.515728 -184.409028 -183.094618 -175.921036 -175.240211 -174.827178 -174.554636 -174.025945 ", 
        "4", 
        "prescribe mainstream bach quest ordering boldly undergraduate pub arena arise voluntarily unlikely migrant collegesand accepting paige verifying civilization grabowicz formats ", 
        "-205.082391 -204.93939 -204.242561 -203.812598 -203.559237 -202.749543 -202.648718 -202.631376 -202.621205 -202.58474 -202.056941 -201.985518 -201.968175 -201.962545 -201.799734 -201.724655 -201.710288 -201.706541 -201.689288 -201.663592 ", 
        "5", 
        "quest trumpeted mainstream verifying undergraduate "
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "superpower ofthe arena zealots migrant civilization boldly pub processing fanatics unlikely prescribe accepting collegesand ordering ", 
        "-178.625867 -178.107447 -177.872185 -177.692374 -177.36237 -177.328732 -177.281571 -176.861845 -176.772052 -176.570098 -175.982638 -175.591856 -175.43708 -175.277383 -175.275297 -175.003181 -174.562818 -174.557168 -174.551971 -174.346415 ", 
        "6", 
        "zealots thaws slices someday shoes layout calve mainstream produces catches quest undergraduate ofthe illustratively seventies migrant ward supremacy permafrost slope ", 
        "-193.340606 -193.193373 -191.761922 -191.244777 -191.163757 -191.084286 -191.056373 -190.982203 -190.837472 -190.696674 -190.5624 -190.561603 -190.553788 -190.540058 -190.471479 -190.26474 -190.191709 -190.150228 -190.093946 -190.088073 ", 
        "7", 
        "superpower undergraduate quest zealots trumpeted mainstream migrant arena fanatics ofthe boldly civilization collegesand supremacy advocating bach norm voluntarily unlikely verifying ", 
        "-176.087704 -175.465218 -174.602681 -174.601029 -174.571017 -173.943107 -173.851154 -173.603583 -173.415421 -173.316831 -172.689322 -170.899139 -170.408582 -169.777039 -169.42945 -164.36777 -164.318796 -163.058363 -162.670338 -162.482242 ", 
        "8", 
        "seventies shoes soybeans produces slope someday layout permafrost calve slices unrest ward thaws plugging inches bioproducts catches blazed calves shoving ", 
        "-196.408967 -195.929051 -195.488543 -195.355893 -194.998169 -194.910638 -194.492089 -194.489075 -194.424074 -194.202122 -193.955816 -193.939117 -193.806897 -193.472615 -193.257587 -193.200973 -192.892608 -192.78294 -192.601746 -192.221893 ", 
        "9", 
        "unlikely reissue permafrost voluntarily pub grabowicz arise calves ordering verifying ward shoes maternity prescribe fontaine seventies produces accepting unrest piper ", 
        "-176.08742 -175.853402 -174.665712 -174.020227 -173.963608 -173.503668 -173.375428 -172.897699 -172.713497 -172.680826 -172.60058 -172.597076 -172.570874 -172.491435 -172.366138 -172.166508 -172.085105 -172.05024 -171.970089 -171.882877 "
       ]
      }
     ], 
     "prompt_number": 165
    }, 
    {
     "cell_type": "code", 
     "collapsed": false, 
     "input": [
      "topics[0][:2]"
     ], 
     "language": "python", 
     "outputs": [
      {
       "output_type": "pyout", 
       "prompt_number": 166, 
       "text": [
        "[(7693, 'superpower', -184.922201), (9511, 'civilization', -183.073103)]"
       ]
      }
     ], 
     "prompt_number": 166
    }, 
    {
     "cell_type": "code", 
     "collapsed": true, 
     "input": [], 
     "language": "python", 
     "outputs": [], 
     "prompt_number": 166
    }, 
    {
     "cell_type": "markdown", 
     "source": [
      "##Predict development set"
     ]
    }, 
    {
     "cell_type": "code", 
     "collapsed": true, 
     "input": [
      "test_data  = 'tmp.sLDA/inf_in/test-data.dat'", 
      "test_label = 'tmp.sLDA/inf_in/test-label.dat'"
     ], 
     "language": "python", 
     "outputs": [], 
     "prompt_number": 167
    }, 
    {
     "cell_type": "code", 
     "collapsed": false, 
     "input": [
      "!mkdir -p tmp.sLDA/inf_in", 
      "!rm tmp.sLDA/inf_in/*", 
      "!mkdir tmp.sLDA/inf_out", 
      "!rm tmp.sLDA/inf_out/*", 
      "with open(test_data,'w') as f:", 
      "    for i,feats in dev.features.iteritems():", 
      "        f.write(str(len(feats))+' ')", 
      "        for w,wid,cnt in feats:", 
      "            f.write(str(wid)+':'+str(cnt)+' ')", 
      "        f.write('\\n')", 
      "!head -n 2 $test_data"
     ], 
     "language": "python", 
     "outputs": [
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "mkdir: tmp.sLDA/inf_out: File exists"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "11 9:2 1827:2 3:1 45:1 18:1 13:1 303:1 157:1 0:1 1:1 24:1 ", 
        "7 6:1 45:1 512:1 9:1 18:1 1068:1 122:1 "
       ]
      }
     ], 
     "prompt_number": 168
    }, 
    {
     "cell_type": "code", 
     "collapsed": false, 
     "input": [
      "with open(test_label,'w') as f:", 
      "    for i,bill in dev.bill.iteritems():", 
      "        f.write(str(bills.index(bill))+'\\n')", 
      "!head -n 2 $test_label"
     ], 
     "language": "python", 
     "outputs": [
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "2", 
        "2"
       ]
      }
     ], 
     "prompt_number": 169
    }, 
    {
     "cell_type": "code", 
     "collapsed": false, 
     "input": [
      "inf_out      = 'tmp.sLDA/inf_out'", 
      "%time !slda inf $test_data $test_label $settings $output/final.model $inf_out"
     ], 
     "language": "python", 
     "outputs": [
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "", 
        "reading data from tmp.sLDA/inf_in/test-data.dat", 
        "number of docs  : 89", 
        "number of terms : 17368", 
        "number of total words : 24920", 
        "", 
        "reading labels from tmp.sLDA/inf_in/test-label.dat", 
        "number of classes : 3", 
        "", 
        "alpha is fixed ...", 
        "var max iter 20", 
        "var convergence 1.00E-03", 
        "em max iter 50", 
        "em convergence 1.00E-04", 
        "L2 penalty 1.00E-02", 
        "", 
        "results will be saved in tmp.sLDA/inf_out", 
        "document 0"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "average accuracy: 0.708"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "CPU times: user 0.01 s, sys: 0.01 s, total: 0.02 s"
       ]
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "", 
        "Wall time: 0.22 s"
       ]
      }
     ], 
     "prompt_number": 170
    }, 
    {
     "cell_type": "code", 
     "collapsed": false, 
     "input": [
      "!ls $inf_out"
     ], 
     "language": "python", 
     "outputs": [
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "inf-gamma.dat      inf-labels.dat     inf-likelihood.dat"
       ]
      }
     ], 
     "prompt_number": 24
    }, 
    {
     "cell_type": "code", 
     "collapsed": false, 
     "input": [
      "!head $inf_out/inf-labels.dat | sed 4q"
     ], 
     "language": "python", 
     "outputs": [
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "2", 
        "1", 
        "1", 
        "0"
       ]
      }
     ], 
     "prompt_number": 27
    }, 
    {
     "cell_type": "code", 
     "collapsed": false, 
     "input": [
      "!head $inf_out/inf-likelihood.dat | sed 4q"
     ], 
     "language": "python", 
     "outputs": [
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "-1780.54938", 
        "-832.24244", 
        "-2012.93747", 
        "-42.75307"
       ]
      }
     ], 
     "prompt_number": 29
    }, 
    {
     "cell_type": "code", 
     "collapsed": false, 
     "input": [
      "!head $inf_out/inf-gamma.dat"
     ], 
     "language": "python", 
     "outputs": [
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": [
        "12.8363576423 10.4441321845 82.3096797782 54.0873970249 11.1621420026 2.0454750005 64.0808835011 8.1890989088 10.7419354743 6.1028984828", 
        "9.0760787709 7.0843038557 2.0271847910 17.4053592363 62.8925461275 1.8086506269 21.1906922185 19.0860530620 2.6672039362 2.7619273749", 
        "11.4647372786 4.9297795488 52.5003992940 9.8143290042 185.6187863843 4.5660634317 23.1678345662 4.9896919250 15.7871306644 13.1612479028", 
        "1.1856001560 1.2204413384 1.2204205221 2.6599142569 1.0802750703 1.0759315439 1.2539777260 1.4705042538 4.6584352246 1.1744999080", 
        "1.8491750000 1.8583963788 1.8215138625 14.4038960965 1.3219215578 1.4221683140 2.5353643501 1.9462666338 1.4787621427 1.3625356639", 
        "3.0722642127 4.4607660536 2.7117139712 54.8104075704 1.7224089087 14.2099328264 147.8765101216 2.0690503239 19.1248672345 7.9420787771", 
        "1.5549165949 1.5610207364 1.3694214740 9.0191822057 1.3358469927 1.2277727311 4.5119966477 1.6639017994 1.3565255114 4.3994153068", 
        "14.4616528876 14.4713869846 8.3222550930 46.9697567885 89.6610908936 12.1190120920 212.5596756753 22.9413001150 14.2751382687 11.2187312016", 
        "81.6511518042 5.2043693931 2.2637399189 5.7825473770 7.5226809720 3.5891038864 280.4366082975 2.6286843318 11.7504578258 9.1706561934", 
        "9.4401908614 132.9858043389 4.8530993595 5.7573481008 4.3525016393 6.8020261333 5.9165334363 3.5764099762 3.5725676398 3.7435185145"
       ]
      }
     ], 
     "prompt_number": 30
    }, 
    {
     "cell_type": "code", 
     "collapsed": true, 
     "input": [], 
     "language": "python", 
     "outputs": []
    }
   ]
  }
 ]
}