深度学习框架Tensorflow学习与应用/11-2 simple_word2vec_文本分类.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 补充\n",
    "\n",
    "* 官网代码：https://github.com/tensorflow/tensorflow/tree/master/tensorflow/examples/tutorials/word2vec\n",
    "\n",
    "* 处理中文数据，需要对中文进行结巴分词\n",
    "\n",
    "\n",
    "参考：https://www.cnblogs.com/go-ahead-wsg/p/12640094.html\n",
    "\n",
    "示例代码是skip-gram的训练方式，这里简单概括一下训练的数据怎么来的：比如，有这么一句话“吃面包么”，经过中文分词后得到“吃 面包 么”，skip-gram会“面包”作为输入数据，“吃”和“么”作为标签数据，像这样，“__面包__”，随机初始化词典embedding矩阵后，采用简单的网络进行预测，当预测值为“吃”及“么”时，loss最小；在大量的数据学习后，输出每个词的embedding矩阵。\n",
    "\n",
    "## 下载问题ERROR\n",
    "运行命令下载很慢，并且报错\n",
    "Exception: Failed to verify text8.zip. Can you get to it with a browser?\n",
    "\n",
    "\n",
    "\n",
    "最好自己下载text8.zip\n",
    "\n",
    "百度网盘：https://pan.baidu.com/share/init?surl=HJ421DYmk4I9KjH9DIQt8w\n",
    "提取码：q8s0\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false,
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "8227112\n"
    },
    {
     "output_type": "error",
     "ename": "Exception",
     "evalue": "Failed to verify text8.zip. Can you get to it with a browser?",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mException\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-14-f646e05742f3>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m     34\u001b[0m     \u001b[1;32mreturn\u001b[0m \u001b[0mfilename\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     35\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 36\u001b[1;33m \u001b[0mfilename\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mmaybe_download\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'text8.zip'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m31344016\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     37\u001b[0m \u001b[1;31m# filename = 'text8.zip'\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     38\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<ipython-input-14-f646e05742f3>\u001b[0m in \u001b[0;36mmaybe_download\u001b[1;34m(filename, expected_bytes)\u001b[0m\n\u001b[0;32m     31\u001b[0m         \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mstatinfo\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mst_size\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     32\u001b[0m         raise Exception(\n\u001b[1;32m---> 33\u001b[1;33m             'Failed to verify ' + filename + '. Can you get to it with a browser?')\n\u001b[0m\u001b[0;32m     34\u001b[0m     \u001b[1;32mreturn\u001b[0m \u001b[0mfilename\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     35\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mException\u001b[0m: Failed to verify text8.zip. Can you get to it with a browser?"
     ]
    }
   ],
   "source": [
    "# encoding=utf8  \n",
    "from __future__ import absolute_import\n",
    "from __future__ import division\n",
    "from __future__ import print_function\n",
    "\n",
    "import collections\n",
    "import math\n",
    "import os\n",
    "import random\n",
    "import zipfile\n",
    "\n",
    "import numpy as np\n",
    "from six.moves import urllib\n",
    "from six.moves import xrange  # pylint: disable=redefined-builtin\n",
    "import tensorflow as tf\n",
    "\n",
    "# Step 1: Download the data.\n",
    "url = 'http://mattmahoney.net/dc/'\n",
    "\n",
    "# 下载数据集\n",
    "def maybe_download(filename, expected_bytes):\n",
    "    \"\"\"Download a file if not present, and make sure it's the right size.\"\"\"\n",
    "    if not os.path.exists(filename):\n",
    "        filename, _ = urllib.request.urlretrieve(url + filename, filename)\n",
    "    # 获取文件相关属性\n",
    "    statinfo = os.stat(filename)\n",
    "    # 比对文件的大小是否正确\n",
    "    if statinfo.st_size == expected_bytes:\n",
    "        print('Found and verified', filename)\n",
    "    else:\n",
    "        print(statinfo.st_size)\n",
    "        raise Exception(\n",
    "            'Failed to verify ' + filename + '. Can you get to it with a browser?')\n",
    "    return filename\n",
    "\n",
    "filename = maybe_download('text8.zip', 31344016)\n",
    "# filename = 'text8.zip'\n",
    "\n",
    "# Read the data into a list of strings.\n",
    "def read_data(filename):\n",
    "    \"\"\"Extract the first file enclosed in a zip file as a list of words\"\"\"\n",
    "    with zipfile.ZipFile(filename) as f:\n",
    "        data = tf.compat.as_str(f.read(f.namelist()[0])).split()\n",
    "    return data\n",
    "\n",
    "# 单词表\n",
    "words = read_data(filename)  #list()列表\n",
    "\n",
    "# Data size\n",
    "print('Data size', len(words))\n",
    "\n",
    "# Step 2: Build the dictionary and replace rare words with UNK token.\n",
    "# 只留高频率的前50000个单词，其他的词都归为UNK\n",
    "vocabulary_size = 50000\n",
    "\n",
    "def build_dataset(words, vocabulary_size):\n",
    "    count = [['UNK', -1]]\n",
    "    # extend追加一个列表\n",
    "    # Counter用来统计每个词出现的次数\n",
    "    # most_common返回一个TopN列表，只留50000个单词包括UNK  \n",
    "    # c = Counter('abracadabra')\n",
    "    # c.most_common()\n",
    "    # [('a', 5), ('r', 2), ('b', 2), ('c', 1), ('d', 1)]\n",
    "    # c.most_common(3)\n",
    "    # [('a', 5), ('r', 2), ('b', 2)]\n",
    "    # 前50000个出现次数最多的词\n",
    "    count.extend(collections.Counter(words).most_common(vocabulary_size - 1))\n",
    "    # 生成 dictionary，词对应编号, word:id(0-49999)\n",
    "    # 词频越高编号越小\n",
    "    dictionary = dict()\n",
    "    for word, _ in count:\n",
    "        dictionary[word] = len(dictionary)\n",
    "    # data把数据集的词都编号\n",
    "    data = list()\n",
    "    unk_count = 0\n",
    "    for word in words:\n",
    "        if word in dictionary:\n",
    "            index = dictionary[word]\n",
    "        else:\n",
    "            index = 0  # dictionary['UNK']\n",
    "            unk_count += 1\n",
    "        data.append(index)\n",
    "    # 记录UNK词的数量\n",
    "    count[0][1] = unk_count\n",
    "    # 编号对应词的字典\n",
    "    reverse_dictionary = dict(zip(dictionary.values(), dictionary.keys()))\n",
    "    return data, count, dictionary, reverse_dictionary\n",
    "\n",
    "# data 数据集，编号形式\n",
    "# count 前50000个出现次数最多的词\n",
    "# dictionary 词对应编号\n",
    "# reverse_dictionary 编号对应词\n",
    "data, count, dictionary, reverse_dictionary = build_dataset(words, vocabulary_size)\n",
    "del words  # Hint to reduce memory.\n",
    "print('Most common words (+UNK)', count[:5]) # 前5词频\n",
    "print('Sample data', data[:10], [reverse_dictionary[i] for i in data[:10]])\n",
    "'''print结果\n",
    "Most common words (+UNK) [['UNK', 418391], ('the', 1061396), ('of', 593677), ('and', 416629), ('one', 411764)]\n",
    "Sample data [5244, 3083, 12, 6, 195, 2, 3134, 46, 59, 156] ['anarchism', 'originated', 'as', 'a', 'term', 'of', 'abuse', 'first', 'used', 'against']\n",
    "'''\n",
    "data_index = 0\n",
    "\n",
    "# Step 3: Function to generate a training batch for the skip-gram model.\n",
    "def generate_batch(batch_size, num_skips, skip_window):\n",
    "    global data_index\n",
    "    assert batch_size % num_skips == 0\n",
    "    assert num_skips <= 2 * skip_window\n",
    "    \n",
    "    batch = np.ndarray(shape=(batch_size), dtype=np.int32)\n",
    "    labels = np.ndarray(shape=(batch_size, 1), dtype=np.int32)\n",
    "    \n",
    "    span = 2 * skip_window + 1  # [ skip_window target skip_window ]\n",
    "    buffer = collections.deque(maxlen=span)\n",
    "    # [ skip_window target skip_window ]\n",
    "            # [ skip_window target skip_window ]\n",
    "                    # [ skip_window target skip_window ]\n",
    "            \n",
    "#     [0 1 2 3 4 5 6 7 8 9 ...]\n",
    "#            t     i  \n",
    "    # 循环3次\n",
    "    for _ in range(span):\n",
    "        buffer.append(data[data_index]) # data_index = 0\n",
    "        data_index = (data_index + 1) % len(data)\n",
    "    # 获取batch和labels\n",
    "    for i in range(batch_size // num_skips):\n",
    "        target = skip_window  # target label at the center of the buffer\n",
    "        targets_to_avoid = [skip_window] # 1\n",
    "        # 循环2次，一个目标单词对应两个上下文单词\n",
    "        for j in range(num_skips):\n",
    "            while target in targets_to_avoid:\n",
    "                # 可能先拿到前面的单词也可能先拿到后面的单词\n",
    "                target = random.randint(0, span - 1) # span=3  # random.randint产生随机整数\n",
    "            targets_to_avoid.append(target)\n",
    "            batch[i * num_skips + j] = buffer[skip_window]\n",
    "            labels[i * num_skips + j, 0] = buffer[target]\n",
    "        buffer.append(data[data_index]) # buffer相当于长度为3的队列，append入队由于长度3的限制，会把队首自动出队\n",
    "        data_index = (data_index + 1) % len(data)\n",
    "    # Backtrack a little bit to avoid skipping words in the end of a batch\n",
    "    # 回溯3个词。因为执行完一个batch的操作之后，data_index会往右多偏移span个位置\n",
    "    data_index = (data_index + len(data) - span) % len(data)\n",
    "    return batch, labels\n",
    "\n",
    "# 打印sample data\n",
    "batch, labels = generate_batch(batch_size=8, num_skips=2, skip_window=1)\n",
    "for i in range(8):\n",
    "    print(batch[i], reverse_dictionary[batch[i]],\n",
    "        '->', labels[i, 0], reverse_dictionary[labels[i, 0]])\n",
    "# 3083 originated -> 5244 anarchism\n",
    "# 3083 originated -> 12 as\n",
    "# 12 as -> 3083 originated\n",
    "# 12 as -> 6 a\n",
    "# 6 a -> 195 term\n",
    "# 6 a -> 12 as\n",
    "# 195 term -> 2 of\n",
    "# 195 term -> 6 a\n",
    "\n",
    "# Step 4: Build and train a skip-gram model.\n",
    "batch_size = 128\n",
    "# 词向量维度\n",
    "embedding_size = 128  # Dimension of the embedding vector.\n",
    "skip_window = 1       # How many words to consider left and right.\n",
    "num_skips = 2         # How many times to reuse an input to generate a label. 生成2次label\n",
    "\n",
    "# We pick a random validation set to sample nearest neighbors. Here we limit the\n",
    "# validation samples to the words that have a low numeric ID, which by\n",
    "# construction are also the most frequent.\n",
    "valid_size = 16     # Random set of words to evaluate similarity on.\n",
    "valid_window = 100  # Only pick dev samples in the head of the distribution.\n",
    "# 从0-100抽取16个整数，无放回抽样\n",
    "valid_examples = np.random.choice(valid_window, valid_size, replace=False) \n",
    "# 负采样样本数\n",
    "num_sampled = 64    # Number of negative examples to sample.\n",
    "\n",
    "graph = tf.Graph()\n",
    "with graph.as_default():\n",
    "    # Input data.\n",
    "    train_inputs = tf.placeholder(tf.int32, shape=[batch_size])\n",
    "    train_labels = tf.placeholder(tf.int32, shape=[batch_size, 1])\n",
    "    valid_dataset = tf.constant(valid_examples, dtype=tf.int32)\n",
    "\n",
    "    # Ops and variables pinned to the CPU because of missing GPU implementation\n",
    "#     with tf.device('/cpu:0'):\n",
    "        # 词向量\n",
    "        # Look up embeddings for inputs.\n",
    "    embeddings = tf.Variable(\n",
    "        tf.random_uniform([vocabulary_size, embedding_size], -1.0, 1.0))\n",
    "    # embedding_lookup(params,ids)其实就是按照ids顺序返回params中的第ids行\n",
    "    # 比如说，ids=[1,7,4],就是返回params中第1,7,4行。返回结果为由params的1,7,4行组成的tensor\n",
    "    # 提取要训练的词\n",
    "    embed = tf.nn.embedding_lookup(embeddings, train_inputs)\n",
    "\n",
    "    # Construct the variables for the noise-contrastive estimation(NCE) loss\n",
    "    nce_weights = tf.Variable(\n",
    "        tf.truncated_normal([vocabulary_size, embedding_size],\n",
    "                        stddev=1.0 / math.sqrt(embedding_size)))\n",
    "    nce_biases = tf.Variable(tf.zeros([vocabulary_size]))\n",
    "\n",
    "    # Compute the average NCE loss for the batch.\n",
    "    # tf.nce_loss automatically draws a new sample of the negative labels each\n",
    "    # time we evaluate the loss.\n",
    "    loss = tf.reduce_mean(\n",
    "        tf.nn.nce_loss(weights=nce_weights,\n",
    "                       biases=nce_biases,\n",
    "                       labels=train_labels,\n",
    "                       inputs=embed,\n",
    "                       num_sampled=num_sampled,   \n",
    "                       num_classes=vocabulary_size))\n",
    "\n",
    "    # Construct the SGD optimizer using a learning rate of 1.0.\n",
    "    optimizer = tf.train.GradientDescentOptimizer(1).minimize(loss)\n",
    "\n",
    "    # Compute the cosine similarity between minibatch examples and all embeddings.\n",
    "    norm = tf.sqrt(tf.reduce_sum(tf.square(embeddings), 1, keep_dims=True))\n",
    "    normalized_embeddings = embeddings / norm\n",
    "    # 抽取一些常用词来测试余弦相似度\n",
    "    valid_embeddings = tf.nn.embedding_lookup(\n",
    "        normalized_embeddings, valid_dataset)\n",
    "    # valid_size == 16\n",
    "    # [16,1] * [1*50000] = [16,50000]\n",
    "    similarity = tf.matmul(\n",
    "        valid_embeddings, normalized_embeddings, transpose_b=True)\n",
    "\n",
    "    # Add variable initializer.\n",
    "    init = tf.global_variables_initializer()\n",
    "\n",
    "# Step 5: Begin training.\n",
    "num_steps = 100001\n",
    "final_embeddings = []\n",
    "\n",
    "with tf.Session(graph=graph) as session:\n",
    "    # We must initialize all variables before we use them.\n",
    "    init.run()\n",
    "    print(\"Initialized\")\n",
    "\n",
    "    average_loss = 0\n",
    "    for step in xrange(num_steps):\n",
    "        # 获取一个批次的target，以及对应的labels，都是编号形式的\n",
    "        batch_inputs, batch_labels = generate_batch(\n",
    "            batch_size, num_skips, skip_window)\n",
    "        feed_dict = {train_inputs: batch_inputs, train_labels: batch_labels}\n",
    "\n",
    "        # We perform one update step by evaluating the optimizer op (including it\n",
    "        # in the list of returned values for session.run()\n",
    "        _, loss_val = session.run([optimizer, loss], feed_dict=feed_dict)\n",
    "        average_loss += loss_val\n",
    "\n",
    "        # 计算训练2000次的平均loss\n",
    "        if step % 2000 == 0:\n",
    "            if step > 0:\n",
    "                average_loss /= 2000\n",
    "            # The average loss is an estimate of the loss over the last 2000 batches.\n",
    "            print(\"Average loss at step \", step, \": \", average_loss)\n",
    "            average_loss = 0\n",
    "    \n",
    "        # Note that this is expensive (~20% slowdown if computed every 500 steps)\n",
    "        if step % 20000 == 0:\n",
    "            sim = similarity.eval()\n",
    "            # 计算验证集的余弦相似度最高的词\n",
    "            for i in xrange(valid_size):\n",
    "                # 根据id拿到对应单词\n",
    "                valid_word = reverse_dictionary[valid_examples[i]]\n",
    "                top_k = 8  # number of nearest neighbors\n",
    "                # 从大到小排序，排除自己本身，取前top_k个值\n",
    "                nearest = (-sim[i, :]).argsort()[1:top_k + 1]\n",
    "                log_str = \"Nearest to %s:\" % valid_word\n",
    "                for k in xrange(top_k):\n",
    "                    close_word = reverse_dictionary[nearest[k]]\n",
    "                    log_str = \"%s %s,\" % (log_str, close_word)\n",
    "                print(log_str)\n",
    "    # 训练结束得到的词向量\n",
    "    final_embeddings = normalized_embeddings.eval()\n",
    "\n",
    "# Step 6: Visualize the embeddings.\n",
    "\n",
    "def plot_with_labels(low_dim_embs, labels, filename='tsne.png'):\n",
    "    assert low_dim_embs.shape[0] >= len(labels), \"More labels than embeddings\"\n",
    "    # 设置图片大小\n",
    "    plt.figure(figsize=(15, 15))  # in inches\n",
    "    for i, label in enumerate(labels):\n",
    "        x, y = low_dim_embs[i, :]\n",
    "        plt.scatter(x, y)\n",
    "        plt.annotate(label,\n",
    "                 xy=(x, y),\n",
    "                 xytext=(5, 2),\n",
    "                 textcoords='offset points',\n",
    "                 ha='right',\n",
    "                 va='bottom')\n",
    "\n",
    "    plt.savefig(filename)\n",
    "\n",
    "try:\n",
    "    from sklearn.manifold import TSNE\n",
    "    import matplotlib.pyplot as plt\n",
    "\n",
    "    tsne = TSNE(perplexity=30, n_components=2, init='pca', n_iter=5000, method='exact')# mac：method='exact'\n",
    "    # 画500个点\n",
    "    plot_only = 500\n",
    "    low_dim_embs = tsne.fit_transform(final_embeddings[:plot_only, :])\n",
    "    labels = [reverse_dictionary[i] for i in xrange(plot_only)]\n",
    "    plot_with_labels(low_dim_embs, labels)\n",
    "\n",
    "except ImportError:\n",
    "    print(\"Please install sklearn, matplotlib, and scipy to visualize embeddings.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nearest to one: two, seven, six, four, microcebus, five, three, eight,\n"
     ]
    }
   ],
   "source": [
    "# with tf.Session() as session:\n",
    "#     valid_word = \"one\"\n",
    "#     valid_examples = dictionary[valid_word]\n",
    "#     valid_dataset = tf.constant([valid_examples], dtype=tf.int32)\n",
    "#     valid_embeddings = tf.nn.embedding_lookup(final_embeddings, valid_dataset)\n",
    "#     similarity = tf.matmul(valid_embeddings, final_embeddings, transpose_b=True)\n",
    "#     sim = similarity.eval()\n",
    "\n",
    "#     top_k = 8  # number of nearest neighbors\n",
    "#     nearest = (-sim[0]).argsort()[1:top_k + 1]# 排除自己本身，从小到大排序\n",
    "#     log_str = \"Nearest to %s:\" % valid_word\n",
    "#     for k in xrange(top_k):\n",
    "#         close_word = reverse_dictionary[nearest[k]]\n",
    "#         log_str = \"%s %s,\" % (log_str, close_word)\n",
    "#     print(log_str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}