05_aligning_the_embeddings_using_vecmap.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In this NB I export embeddings to a word2vec format. I use [vecmap](https://github.com/artetxem/vecmap) from console to perform the mapping and evaluate the results. I use [annoy](https://github.com/spotify/annoy) to find nearest neighbors and use our evaluation functionality to evaluate performance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export data\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from decoder_head.core import *\n",
    "from decoder_head.data import *\n",
    "from fastai2.text.all import *\n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "vocab_en= make_vocab(pd.read_pickle('data/en-100_tok/counter.pkl'), max_vocab=4000)\n",
    "vocab_es= make_vocab(pd.read_pickle('data/es-100_tok/counter.pkl'), max_vocab=4000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = 'data/en-100_tok/'\n",
    "mult = 4\n",
    "bs = 80\n",
    "seq_len = 70\n",
    "\n",
    "lm = DataBlock(blocks=(TextBlock(vocab=vocab_en, is_lm=True),),\n",
    "                get_x=read_tokenized_file,\n",
    "                get_items=partial(get_text_files, folders=['train', 'valid']),\n",
    "                splitter=FuncSplitter(lambda itm: itm.parent.name == 'valid'))\n",
    "\n",
    "dbunch_lm = lm.databunch(path, path=path, bs=bs, seq_len=seq_len)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn = language_model_learner(\n",
    "    dbunch_lm,\n",
    "    p_normAWD_LSTM,\n",
    "    opt_func=opt,\n",
    "    pretrained=False,\n",
    "    config=awd_lstm_lm_config,\n",
    "    drop_mult=0.1,\n",
    "    metrics=[accuracy, top_k_accuracy]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<fastai2.text.learner.LMLearner at 0x7f69535f6fd0>"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn.load('emb_norm_rows_columns') # en LM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4008"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(vocab_en)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export data\n",
    "def embs_to_txt(vocab, embeddings, fname):\n",
    "    '''writes embeddings to txt file in word2vec format'''\n",
    "    lines = []\n",
    "    lines.append(f'{len(vocab)} {embeddings.shape[1]}\\n')\n",
    "    for word, t in zip(vocab, embeddings):\n",
    "        word = re.subn('\\n', '', word)[0]\n",
    "        lines.append(f\"{word} {' '.join([str(datum.item()) for datum in t])}\\n\")\n",
    "    with open(fname, 'w') as f:\n",
    "        f.writelines(lines)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "embs_to_txt(vocab_en, learn.model[0].encoder.normalized_weight(), 'data/en_norm_embs.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = 'data/es-100_tok/'\n",
    "mult = 4\n",
    "bs = 80\n",
    "seq_len = 70\n",
    "\n",
    "lm = DataBlock(blocks=(TextBlock(vocab=vocab_es, is_lm=True),),\n",
    "                get_x=read_tokenized_file,\n",
    "                get_items=partial(get_text_files, folders=['train', 'valid']),\n",
    "                splitter=FuncSplitter(lambda itm: itm.parent.name == 'valid'))\n",
    "\n",
    "dbunch_lm = lm.databunch(path, path=path, bs=bs, seq_len=seq_len)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn = language_model_learner(\n",
    "    dbunch_lm,\n",
    "    p_normAWD_LSTM,\n",
    "    opt_func=opt,\n",
    "    pretrained=False,\n",
    "    config=awd_lstm_lm_config,\n",
    "    drop_mult=0.1,\n",
    "    metrics=[accuracy, top_k_accuracy]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<fastai2.text.learner.LMLearner at 0x7f6952844490>"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn.load('pLSTM_es')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "embs_to_txt(vocab_en, learn.model[0].encoder.normalized_weight(), 'data/es_norm_embs.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# python map_embeddings.py --cuda --unsupervised ~/workspace/decoder_head/data/en_norm_embs.txt ~/workspace/decoder_head/data/es_norm_embs.txt ~/workspace/decoder_head/data/en_src.txt ~/workspace/decoder_head/data/es_trg.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "en_es_dict = get_en_es_dict()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "import annoy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('data/en_src.txt') as f:\n",
    "    en_src = f.readlines()\n",
    "    \n",
    "with open('data/es_trg.txt') as f:\n",
    "    es_trg = f.readlines()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "en_embs = []\n",
    "for l in en_src[1:]:\n",
    "    en_embs.append([float(s) for s in l.split()[1:]])\n",
    "\n",
    "en_embs = np.array(en_embs)\n",
    "\n",
    "es_embs = []\n",
    "for l in es_trg[1:]:\n",
    "    es_embs.append([float(s) for s in l.split()[1:]])\n",
    "\n",
    "es_embs = np.array(es_embs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "# No need to perform the normalization - the embeddings are already normalized!\n",
    "\n",
    "# en_embs_norm = en_embs / np.linalg.norm(en_embs, axis=1)[:, None]\n",
    "# es_embs_norm = es_embs / np.linalg.norm(es_embs, axis=1)[:, None]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "from annoy import AnnoyIndex"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t = AnnoyIndex(100, 'euclidean')\n",
    "\n",
    "for i in range(len(en_embs_norm)):\n",
    "    t.add_item(i, en_embs_norm[i])\n",
    "    \n",
    "t.build(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.3908608731130151"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "top_n_translation_acc(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5091799265605875"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "top_n_translation_acc(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.551203590371277"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "top_n_translation_acc(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6597307221542228"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "top_n_translation_acc(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This is a very valuable piece of information to us. First of all, our embeddings contain enough structure to faciliate the alignment. Secondly, the evaluation framework seems to work.\n",
    "\n",
    "Given this result, I attempted translation using decoder head with hinting. What I mean by that, is I trained an English and a Spanish LM but using already aligned embeddings! As expected, they trained with no issues. I include the only bit of additional code I had to write below (for loading the embeddings from txt)\n",
    "\n",
    "Unfortunately, despite beginning from this hinted state, there is no meaningful improvement on the translation task.\n",
    "\n",
    "The two observation I have made while working on this, that I believe could be important to consider when devising next steps, are as follows.\n",
    "\n",
    "First of all, with the permutation matrix, we only allow the model to learn linear combinations of embeddings. This can only work if the column semantics between embeddings from different LMs are aligned. If col 0 of English LM encodes the abstract concept of hot - cold, the Spanish LM would need to also assign the same meaning to col 0. I do understand that there is a mathematical way to reason about this in terms of the topography of the embedding cloud, but it is easier to reason about this and especially to explain the reasoning using a concrete example as above (even if it is not an entirely rigorous treatment).\n",
    "\n",
    "Secondly, even with mapped embeddings, the model is unable to recover the mapping of embeddings to ids. It is important to note that it finds a solution that is quite good (~28% LM accuracy without hinting, 35% LM accuracy with hinting), but the linear combination of embeddings it finds to lower the loss is nonsensical in terms of translation ability."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export data\n",
    "\n",
    "def txt_to_embs(fname):\n",
    "    with open(fname, 'r') as f:\n",
    "        lines = f.readlines()\n",
    "    vocab = []\n",
    "    embs = []\n",
    "    for line in lines[1:]:\n",
    "        l = line.split()\n",
    "        vocab.append(l[0])\n",
    "        embs.append(np.array([float(s) for s in l[1:]]))\n",
    "    return vocab, np.stack(embs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Converted 00_data.ipynb.\n",
      "Converted 01_train_LM_en.ipynb.\n",
      "Converted 02_train_LM_es.ipynb.\n",
      "Converted 03_translate_en_to_es.ipynb.\n",
      "Converted 04_LM_with_normalized_embeddings.ipynb.\n",
      "Converted 04a_LM_with_normalized_embeddings_mixer_softmax.ipynb.\n",
      "Converted 05_aligning_the_embeddings_using_vecmap.ipynb.\n",
      "Converted 99_index.ipynb.\n"
     ]
    }
   ],
   "source": [
    "#hide\n",
    "from nbdev.export import notebook2script\n",
    "notebook2script()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}