parallel-sentence-aligner.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "d005c107",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: tensorflow==2.5.0 in ./.pyenv/lib/python3.7/site-packages (2.5.0)\n",
      "Requirement already satisfied: google-pasta~=0.2 in ./.pyenv/lib/python3.7/site-packages (from tensorflow==2.5.0) (0.2.0)\n",
      "Requirement already satisfied: keras-nightly~=2.5.0.dev in ./.pyenv/lib/python3.7/site-packages (from tensorflow==2.5.0) (2.5.0.dev2021032900)\n",
      "Requirement already satisfied: opt-einsum~=3.3.0 in ./.pyenv/lib/python3.7/site-packages (from tensorflow==2.5.0) (3.3.0)\n",
      "Requirement already satisfied: astunparse~=1.6.3 in ./.pyenv/lib/python3.7/site-packages (from tensorflow==2.5.0) (1.6.3)\n",
      "Requirement already satisfied: wrapt~=1.12.1 in ./.pyenv/lib/python3.7/site-packages (from tensorflow==2.5.0) (1.12.1)\n",
      "Requirement already satisfied: wheel~=0.35 in ./.pyenv/lib/python3.7/site-packages (from tensorflow==2.5.0) (0.36.2)\n",
      "Requirement already satisfied: numpy~=1.19.2 in ./.pyenv/lib/python3.7/site-packages (from tensorflow==2.5.0) (1.19.5)\n",
      "Requirement already satisfied: six~=1.15.0 in ./.pyenv/lib/python3.7/site-packages (from tensorflow==2.5.0) (1.15.0)\n",
      "Requirement already satisfied: tensorflow-estimator<2.6.0,>=2.5.0rc0 in ./.pyenv/lib/python3.7/site-packages (from tensorflow==2.5.0) (2.5.0)\n",
      "Requirement already satisfied: protobuf>=3.9.2 in ./.pyenv/lib/python3.7/site-packages (from tensorflow==2.5.0) (3.17.2)\n",
      "Requirement already satisfied: flatbuffers~=1.12.0 in ./.pyenv/lib/python3.7/site-packages (from tensorflow==2.5.0) (1.12)\n",
      "Requirement already satisfied: keras-preprocessing~=1.1.2 in ./.pyenv/lib/python3.7/site-packages (from tensorflow==2.5.0) (1.1.2)\n",
      "Requirement already satisfied: tensorboard~=2.5 in ./.pyenv/lib/python3.7/site-packages (from tensorflow==2.5.0) (2.5.0)\n",
      "Requirement already satisfied: typing-extensions~=3.7.4 in ./.pyenv/lib/python3.7/site-packages (from tensorflow==2.5.0) (3.7.4.3)\n",
      "Requirement already satisfied: h5py~=3.1.0 in ./.pyenv/lib/python3.7/site-packages (from tensorflow==2.5.0) (3.1.0)\n",
      "Requirement already satisfied: absl-py~=0.10 in ./.pyenv/lib/python3.7/site-packages (from tensorflow==2.5.0) (0.12.0)\n",
      "Requirement already satisfied: gast==0.4.0 in ./.pyenv/lib/python3.7/site-packages (from tensorflow==2.5.0) (0.4.0)\n",
      "Requirement already satisfied: grpcio~=1.34.0 in ./.pyenv/lib/python3.7/site-packages (from tensorflow==2.5.0) (1.34.1)\n",
      "Requirement already satisfied: termcolor~=1.1.0 in ./.pyenv/lib/python3.7/site-packages (from tensorflow==2.5.0) (1.1.0)\n",
      "Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in ./.pyenv/lib/python3.7/site-packages (from tensorboard~=2.5->tensorflow==2.5.0) (0.4.4)\n",
      "Requirement already satisfied: tensorboard-data-server<0.7.0,>=0.6.0 in ./.pyenv/lib/python3.7/site-packages (from tensorboard~=2.5->tensorflow==2.5.0) (0.6.1)\n",
      "Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in ./.pyenv/lib/python3.7/site-packages (from tensorboard~=2.5->tensorflow==2.5.0) (1.8.0)\n",
      "Requirement already satisfied: markdown>=2.6.8 in ./.pyenv/lib/python3.7/site-packages (from tensorboard~=2.5->tensorflow==2.5.0) (3.3.4)\n",
      "Requirement already satisfied: google-auth<2,>=1.6.3 in ./.pyenv/lib/python3.7/site-packages (from tensorboard~=2.5->tensorflow==2.5.0) (1.30.1)\n",
      "Requirement already satisfied: werkzeug>=0.11.15 in ./.pyenv/lib/python3.7/site-packages (from tensorboard~=2.5->tensorflow==2.5.0) (2.0.1)\n",
      "Requirement already satisfied: setuptools>=41.0.0 in ./.pyenv/lib/python3.7/site-packages (from tensorboard~=2.5->tensorflow==2.5.0) (41.2.0)\n",
      "Requirement already satisfied: requests<3,>=2.21.0 in ./.pyenv/lib/python3.7/site-packages (from tensorboard~=2.5->tensorflow==2.5.0) (2.25.1)\n",
      "Requirement already satisfied: cached-property; python_version < \"3.8\" in ./.pyenv/lib/python3.7/site-packages (from h5py~=3.1.0->tensorflow==2.5.0) (1.5.2)\n",
      "Requirement already satisfied: requests-oauthlib>=0.7.0 in ./.pyenv/lib/python3.7/site-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard~=2.5->tensorflow==2.5.0) (1.3.0)\n",
      "Requirement already satisfied: importlib-metadata; python_version < \"3.8\" in ./.pyenv/lib/python3.7/site-packages (from markdown>=2.6.8->tensorboard~=2.5->tensorflow==2.5.0) (4.4.0)\n",
      "Requirement already satisfied: rsa<5,>=3.1.4; python_version >= \"3.6\" in ./.pyenv/lib/python3.7/site-packages (from google-auth<2,>=1.6.3->tensorboard~=2.5->tensorflow==2.5.0) (4.7.2)\n",
      "Requirement already satisfied: cachetools<5.0,>=2.0.0 in ./.pyenv/lib/python3.7/site-packages (from google-auth<2,>=1.6.3->tensorboard~=2.5->tensorflow==2.5.0) (4.2.2)\n",
      "Requirement already satisfied: pyasn1-modules>=0.2.1 in ./.pyenv/lib/python3.7/site-packages (from google-auth<2,>=1.6.3->tensorboard~=2.5->tensorflow==2.5.0) (0.2.8)\n",
      "Requirement already satisfied: chardet<5,>=3.0.2 in ./.pyenv/lib/python3.7/site-packages (from requests<3,>=2.21.0->tensorboard~=2.5->tensorflow==2.5.0) (4.0.0)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in ./.pyenv/lib/python3.7/site-packages (from requests<3,>=2.21.0->tensorboard~=2.5->tensorflow==2.5.0) (2021.5.30)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in ./.pyenv/lib/python3.7/site-packages (from requests<3,>=2.21.0->tensorboard~=2.5->tensorflow==2.5.0) (1.26.5)\n",
      "Requirement already satisfied: idna<3,>=2.5 in ./.pyenv/lib/python3.7/site-packages (from requests<3,>=2.21.0->tensorboard~=2.5->tensorflow==2.5.0) (2.10)\n",
      "Requirement already satisfied: oauthlib>=3.0.0 in ./.pyenv/lib/python3.7/site-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard~=2.5->tensorflow==2.5.0) (3.1.1)\n",
      "Requirement already satisfied: zipp>=0.5 in ./.pyenv/lib/python3.7/site-packages (from importlib-metadata; python_version < \"3.8\"->markdown>=2.6.8->tensorboard~=2.5->tensorflow==2.5.0) (3.4.1)\n",
      "Requirement already satisfied: pyasn1>=0.1.3 in ./.pyenv/lib/python3.7/site-packages (from rsa<5,>=3.1.4; python_version >= \"3.6\"->google-auth<2,>=1.6.3->tensorboard~=2.5->tensorflow==2.5.0) (0.4.8)\n",
      "\u001b[33mWARNING: You are using pip version 19.2.3, however version 21.1.2 is available.\n",
      "You should consider upgrading via the 'pip install --upgrade pip' command.\u001b[0m\n",
      "Requirement already satisfied: tensorflow-hub==0.8.0 in ./.pyenv/lib/python3.7/site-packages (0.8.0)\n",
      "Requirement already satisfied: numpy>=1.12.0 in ./.pyenv/lib/python3.7/site-packages (from tensorflow-hub==0.8.0) (1.19.5)\n",
      "Requirement already satisfied: protobuf>=3.8.0 in ./.pyenv/lib/python3.7/site-packages (from tensorflow-hub==0.8.0) (3.17.2)\n",
      "Requirement already satisfied: six>=1.12.0 in ./.pyenv/lib/python3.7/site-packages (from tensorflow-hub==0.8.0) (1.15.0)\n",
      "\u001b[33mWARNING: You are using pip version 19.2.3, however version 21.1.2 is available.\n",
      "You should consider upgrading via the 'pip install --upgrade pip' command.\u001b[0m\n",
      "Requirement already satisfied: tensorflow_text==2.5.0 in ./.pyenv/lib/python3.7/site-packages (2.5.0)\n",
      "Requirement already satisfied: tensorflow-hub>=0.8.0 in ./.pyenv/lib/python3.7/site-packages (from tensorflow_text==2.5.0) (0.8.0)\n",
      "Requirement already satisfied: tensorflow<2.6,>=2.5.0 in ./.pyenv/lib/python3.7/site-packages (from tensorflow_text==2.5.0) (2.5.0)\n",
      "Requirement already satisfied: protobuf>=3.8.0 in ./.pyenv/lib/python3.7/site-packages (from tensorflow-hub>=0.8.0->tensorflow_text==2.5.0) (3.17.2)\n",
      "Requirement already satisfied: numpy>=1.12.0 in ./.pyenv/lib/python3.7/site-packages (from tensorflow-hub>=0.8.0->tensorflow_text==2.5.0) (1.19.5)\n",
      "Requirement already satisfied: six>=1.12.0 in ./.pyenv/lib/python3.7/site-packages (from tensorflow-hub>=0.8.0->tensorflow_text==2.5.0) (1.15.0)\n",
      "Requirement already satisfied: keras-preprocessing~=1.1.2 in ./.pyenv/lib/python3.7/site-packages (from tensorflow<2.6,>=2.5.0->tensorflow_text==2.5.0) (1.1.2)\n",
      "Requirement already satisfied: wrapt~=1.12.1 in ./.pyenv/lib/python3.7/site-packages (from tensorflow<2.6,>=2.5.0->tensorflow_text==2.5.0) (1.12.1)\n",
      "Requirement already satisfied: typing-extensions~=3.7.4 in ./.pyenv/lib/python3.7/site-packages (from tensorflow<2.6,>=2.5.0->tensorflow_text==2.5.0) (3.7.4.3)\n",
      "Requirement already satisfied: h5py~=3.1.0 in ./.pyenv/lib/python3.7/site-packages (from tensorflow<2.6,>=2.5.0->tensorflow_text==2.5.0) (3.1.0)\n",
      "Requirement already satisfied: gast==0.4.0 in ./.pyenv/lib/python3.7/site-packages (from tensorflow<2.6,>=2.5.0->tensorflow_text==2.5.0) (0.4.0)\n",
      "Requirement already satisfied: grpcio~=1.34.0 in ./.pyenv/lib/python3.7/site-packages (from tensorflow<2.6,>=2.5.0->tensorflow_text==2.5.0) (1.34.1)\n",
      "Requirement already satisfied: termcolor~=1.1.0 in ./.pyenv/lib/python3.7/site-packages (from tensorflow<2.6,>=2.5.0->tensorflow_text==2.5.0) (1.1.0)\n",
      "Requirement already satisfied: flatbuffers~=1.12.0 in ./.pyenv/lib/python3.7/site-packages (from tensorflow<2.6,>=2.5.0->tensorflow_text==2.5.0) (1.12)\n",
      "Requirement already satisfied: opt-einsum~=3.3.0 in ./.pyenv/lib/python3.7/site-packages (from tensorflow<2.6,>=2.5.0->tensorflow_text==2.5.0) (3.3.0)\n",
      "Requirement already satisfied: wheel~=0.35 in ./.pyenv/lib/python3.7/site-packages (from tensorflow<2.6,>=2.5.0->tensorflow_text==2.5.0) (0.36.2)\n",
      "Requirement already satisfied: tensorflow-estimator<2.6.0,>=2.5.0rc0 in ./.pyenv/lib/python3.7/site-packages (from tensorflow<2.6,>=2.5.0->tensorflow_text==2.5.0) (2.5.0)\n",
      "Requirement already satisfied: absl-py~=0.10 in ./.pyenv/lib/python3.7/site-packages (from tensorflow<2.6,>=2.5.0->tensorflow_text==2.5.0) (0.12.0)\n",
      "Requirement already satisfied: tensorboard~=2.5 in ./.pyenv/lib/python3.7/site-packages (from tensorflow<2.6,>=2.5.0->tensorflow_text==2.5.0) (2.5.0)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: google-pasta~=0.2 in ./.pyenv/lib/python3.7/site-packages (from tensorflow<2.6,>=2.5.0->tensorflow_text==2.5.0) (0.2.0)\n",
      "Requirement already satisfied: astunparse~=1.6.3 in ./.pyenv/lib/python3.7/site-packages (from tensorflow<2.6,>=2.5.0->tensorflow_text==2.5.0) (1.6.3)\n",
      "Requirement already satisfied: keras-nightly~=2.5.0.dev in ./.pyenv/lib/python3.7/site-packages (from tensorflow<2.6,>=2.5.0->tensorflow_text==2.5.0) (2.5.0.dev2021032900)\n",
      "Requirement already satisfied: cached-property; python_version < \"3.8\" in ./.pyenv/lib/python3.7/site-packages (from h5py~=3.1.0->tensorflow<2.6,>=2.5.0->tensorflow_text==2.5.0) (1.5.2)\n",
      "Requirement already satisfied: requests<3,>=2.21.0 in ./.pyenv/lib/python3.7/site-packages (from tensorboard~=2.5->tensorflow<2.6,>=2.5.0->tensorflow_text==2.5.0) (2.25.1)\n",
      "Requirement already satisfied: tensorboard-data-server<0.7.0,>=0.6.0 in ./.pyenv/lib/python3.7/site-packages (from tensorboard~=2.5->tensorflow<2.6,>=2.5.0->tensorflow_text==2.5.0) (0.6.1)\n",
      "Requirement already satisfied: setuptools>=41.0.0 in ./.pyenv/lib/python3.7/site-packages (from tensorboard~=2.5->tensorflow<2.6,>=2.5.0->tensorflow_text==2.5.0) (41.2.0)\n",
      "Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in ./.pyenv/lib/python3.7/site-packages (from tensorboard~=2.5->tensorflow<2.6,>=2.5.0->tensorflow_text==2.5.0) (1.8.0)\n",
      "Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in ./.pyenv/lib/python3.7/site-packages (from tensorboard~=2.5->tensorflow<2.6,>=2.5.0->tensorflow_text==2.5.0) (0.4.4)\n",
      "Requirement already satisfied: google-auth<2,>=1.6.3 in ./.pyenv/lib/python3.7/site-packages (from tensorboard~=2.5->tensorflow<2.6,>=2.5.0->tensorflow_text==2.5.0) (1.30.1)\n",
      "Requirement already satisfied: werkzeug>=0.11.15 in ./.pyenv/lib/python3.7/site-packages (from tensorboard~=2.5->tensorflow<2.6,>=2.5.0->tensorflow_text==2.5.0) (2.0.1)\n",
      "Requirement already satisfied: markdown>=2.6.8 in ./.pyenv/lib/python3.7/site-packages (from tensorboard~=2.5->tensorflow<2.6,>=2.5.0->tensorflow_text==2.5.0) (3.3.4)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in ./.pyenv/lib/python3.7/site-packages (from requests<3,>=2.21.0->tensorboard~=2.5->tensorflow<2.6,>=2.5.0->tensorflow_text==2.5.0) (1.26.5)\n",
      "Requirement already satisfied: chardet<5,>=3.0.2 in ./.pyenv/lib/python3.7/site-packages (from requests<3,>=2.21.0->tensorboard~=2.5->tensorflow<2.6,>=2.5.0->tensorflow_text==2.5.0) (4.0.0)\n",
      "Requirement already satisfied: idna<3,>=2.5 in ./.pyenv/lib/python3.7/site-packages (from requests<3,>=2.21.0->tensorboard~=2.5->tensorflow<2.6,>=2.5.0->tensorflow_text==2.5.0) (2.10)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in ./.pyenv/lib/python3.7/site-packages (from requests<3,>=2.21.0->tensorboard~=2.5->tensorflow<2.6,>=2.5.0->tensorflow_text==2.5.0) (2021.5.30)\n",
      "Requirement already satisfied: requests-oauthlib>=0.7.0 in ./.pyenv/lib/python3.7/site-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard~=2.5->tensorflow<2.6,>=2.5.0->tensorflow_text==2.5.0) (1.3.0)\n",
      "Requirement already satisfied: pyasn1-modules>=0.2.1 in ./.pyenv/lib/python3.7/site-packages (from google-auth<2,>=1.6.3->tensorboard~=2.5->tensorflow<2.6,>=2.5.0->tensorflow_text==2.5.0) (0.2.8)\n",
      "Requirement already satisfied: rsa<5,>=3.1.4; python_version >= \"3.6\" in ./.pyenv/lib/python3.7/site-packages (from google-auth<2,>=1.6.3->tensorboard~=2.5->tensorflow<2.6,>=2.5.0->tensorflow_text==2.5.0) (4.7.2)\n",
      "Requirement already satisfied: cachetools<5.0,>=2.0.0 in ./.pyenv/lib/python3.7/site-packages (from google-auth<2,>=1.6.3->tensorboard~=2.5->tensorflow<2.6,>=2.5.0->tensorflow_text==2.5.0) (4.2.2)\n",
      "Requirement already satisfied: importlib-metadata; python_version < \"3.8\" in ./.pyenv/lib/python3.7/site-packages (from markdown>=2.6.8->tensorboard~=2.5->tensorflow<2.6,>=2.5.0->tensorflow_text==2.5.0) (4.4.0)\n",
      "Requirement already satisfied: oauthlib>=3.0.0 in ./.pyenv/lib/python3.7/site-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard~=2.5->tensorflow<2.6,>=2.5.0->tensorflow_text==2.5.0) (3.1.1)\n",
      "Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in ./.pyenv/lib/python3.7/site-packages (from pyasn1-modules>=0.2.1->google-auth<2,>=1.6.3->tensorboard~=2.5->tensorflow<2.6,>=2.5.0->tensorflow_text==2.5.0) (0.4.8)\n",
      "Requirement already satisfied: zipp>=0.5 in ./.pyenv/lib/python3.7/site-packages (from importlib-metadata; python_version < \"3.8\"->markdown>=2.6.8->tensorboard~=2.5->tensorflow<2.6,>=2.5.0->tensorflow_text==2.5.0) (3.4.1)\n",
      "\u001b[33mWARNING: You are using pip version 19.2.3, however version 21.1.2 is available.\n",
      "You should consider upgrading via the 'pip install --upgrade pip' command.\u001b[0m\n",
      "Requirement already satisfied: spacy==3.0.6 in ./.pyenv/lib/python3.7/site-packages (3.0.6)\n",
      "Requirement already satisfied: jinja2 in ./.pyenv/lib/python3.7/site-packages (from spacy==3.0.6) (3.0.1)\n",
      "Requirement already satisfied: typing-extensions<4.0.0.0,>=3.7.4; python_version < \"3.8\" in ./.pyenv/lib/python3.7/site-packages (from spacy==3.0.6) (3.7.4.3)\n",
      "Requirement already satisfied: numpy>=1.15.0 in ./.pyenv/lib/python3.7/site-packages (from spacy==3.0.6) (1.19.5)\n",
      "Requirement already satisfied: wasabi<1.1.0,>=0.8.1 in ./.pyenv/lib/python3.7/site-packages (from spacy==3.0.6) (0.8.2)\n",
      "Requirement already satisfied: blis<0.8.0,>=0.4.0 in ./.pyenv/lib/python3.7/site-packages (from spacy==3.0.6) (0.7.4)\n",
      "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in ./.pyenv/lib/python3.7/site-packages (from spacy==3.0.6) (2.0.5)\n",
      "Requirement already satisfied: preshed<3.1.0,>=3.0.2 in ./.pyenv/lib/python3.7/site-packages (from spacy==3.0.6) (3.0.5)\n",
      "Requirement already satisfied: setuptools in ./.pyenv/lib/python3.7/site-packages (from spacy==3.0.6) (41.2.0)\n",
      "Requirement already satisfied: packaging>=20.0 in ./.pyenv/lib/python3.7/site-packages (from spacy==3.0.6) (20.9)\n",
      "Requirement already satisfied: requests<3.0.0,>=2.13.0 in ./.pyenv/lib/python3.7/site-packages (from spacy==3.0.6) (2.25.1)\n",
      "Requirement already satisfied: pydantic<1.8.0,>=1.7.1 in ./.pyenv/lib/python3.7/site-packages (from spacy==3.0.6) (1.7.4)\n",
      "Requirement already satisfied: catalogue<2.1.0,>=2.0.3 in ./.pyenv/lib/python3.7/site-packages (from spacy==3.0.6) (2.0.4)\n",
      "Requirement already satisfied: pathy>=0.3.5 in ./.pyenv/lib/python3.7/site-packages (from spacy==3.0.6) (0.5.2)\n",
      "Requirement already satisfied: thinc<8.1.0,>=8.0.3 in ./.pyenv/lib/python3.7/site-packages (from spacy==3.0.6) (8.0.3)\n",
      "Requirement already satisfied: typer<0.4.0,>=0.3.0 in ./.pyenv/lib/python3.7/site-packages (from spacy==3.0.6) (0.3.2)\n",
      "Requirement already satisfied: srsly<3.0.0,>=2.4.1 in ./.pyenv/lib/python3.7/site-packages (from spacy==3.0.6) (2.4.1)\n",
      "Requirement already satisfied: spacy-legacy<3.1.0,>=3.0.4 in ./.pyenv/lib/python3.7/site-packages (from spacy==3.0.6) (3.0.5)\n",
      "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in ./.pyenv/lib/python3.7/site-packages (from spacy==3.0.6) (1.0.5)\n",
      "Requirement already satisfied: tqdm<5.0.0,>=4.38.0 in ./.pyenv/lib/python3.7/site-packages (from spacy==3.0.6) (4.61.0)\n",
      "Requirement already satisfied: MarkupSafe>=2.0 in ./.pyenv/lib/python3.7/site-packages (from jinja2->spacy==3.0.6) (2.0.1)\n",
      "Requirement already satisfied: pyparsing>=2.0.2 in ./.pyenv/lib/python3.7/site-packages (from packaging>=20.0->spacy==3.0.6) (2.4.7)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in ./.pyenv/lib/python3.7/site-packages (from requests<3.0.0,>=2.13.0->spacy==3.0.6) (2021.5.30)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in ./.pyenv/lib/python3.7/site-packages (from requests<3.0.0,>=2.13.0->spacy==3.0.6) (1.26.5)\n",
      "Requirement already satisfied: chardet<5,>=3.0.2 in ./.pyenv/lib/python3.7/site-packages (from requests<3.0.0,>=2.13.0->spacy==3.0.6) (4.0.0)\n",
      "Requirement already satisfied: idna<3,>=2.5 in ./.pyenv/lib/python3.7/site-packages (from requests<3.0.0,>=2.13.0->spacy==3.0.6) (2.10)\n",
      "Requirement already satisfied: zipp>=0.5; python_version < \"3.8\" in ./.pyenv/lib/python3.7/site-packages (from catalogue<2.1.0,>=2.0.3->spacy==3.0.6) (3.4.1)\n",
      "Requirement already satisfied: smart-open<4.0.0,>=2.2.0 in ./.pyenv/lib/python3.7/site-packages (from pathy>=0.3.5->spacy==3.0.6) (3.0.0)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: click<7.2.0,>=7.1.1 in ./.pyenv/lib/python3.7/site-packages (from typer<0.4.0,>=0.3.0->spacy==3.0.6) (7.1.2)\n",
      "\u001b[33mWARNING: You are using pip version 19.2.3, however version 21.1.2 is available.\n",
      "You should consider upgrading via the 'pip install --upgrade pip' command.\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "!pip install tensorflow==2.5.0\n",
    "!pip install tensorflow-hub==0.8.0\n",
    "!pip install tensorflow_text==2.5.0\n",
    "!pip install spacy==3.0.6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "11eb90d8",
   "metadata": {},
   "outputs": [],
   "source": [
    "from spacy.lang.en import English\n",
    "import tensorflow_hub as hub\n",
    "import numpy as np\n",
    "import tensorflow_text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "ceaa415a",
   "metadata": {},
   "outputs": [],
   "source": [
    "en_text = \"\"\"\n",
    "categorical data\n",
    "Features having a discrete set of possible values. Blahblah. \n",
    "\n",
    "For example, consider a categorical feature named house style, \n",
    "which has a discrete set of three possible values: Tudor, ranch, \n",
    "colonial. By representing house style as categorical data, the\n",
    "model can learn the separate impacts of Tudor, ranch, and colonial\n",
    "on house price. Sometimes, values in the discrete set are mutually\n",
    "exclusive, and only one value can be applied to a given example.\n",
    "For example, a car maker categorical feature would probably permit\n",
    "only a single value (Toyota) per example. Other times, more than\n",
    "one value may be applicable. A single car could be painted more\n",
    "than one different color, so a car color categorical feature would\n",
    "likely permit a single example to have multiple values (for\n",
    "example, red and white). Categorical features are sometimes called\n",
    "discrete features.\n",
    "\n",
    "Contrast with numerical data\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "96c443ab",
   "metadata": {},
   "outputs": [],
   "source": [
    "zh_text = \"\"\"\n",
    "分类数据\n",
    "\n",
    "一种特征，拥有一组离散的可能值。\n",
    "\n",
    "以某个名为 house style 的分类特征为例，该特征拥有一组离散的可能值（共三个），\n",
    "即 Tudor, ranch, colonial。通过将 house style 表示成分类数据，相应模型可以\n",
    "学习 Tudor、ranch 和 colonial 分别对房价的影响。有时，离散集中的值是互斥的，\n",
    "只能将其中一个值应用于指定样本。例如，car maker 分类特征可能只允许一个样本有\n",
    "一个值 (Toyota)。你好。在其他情况下，则可以应用多个值。一辆车可能会被喷涂多种\n",
    "不同的颜色，因此，car color 分类特征可能会允许单个样本具有多个值（例如 red 和\n",
    "white）。分类特征有时称为离散特征。\n",
    "\n",
    "与数值数据相对。\n",
    "完\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "0c800e75",
   "metadata": {},
   "outputs": [],
   "source": [
    "def sentencize(text):\n",
    "    text = text.replace('。', '。 ')  # spaCy sentencizer only works when there's space after punctuation\n",
    "    sents = []\n",
    "    nlp = English()\n",
    "    nlp.add_pipe(\"sentencizer\")\n",
    "    doc = nlp(text)\n",
    "    for sent in doc.sents:\n",
    "        sents.append(sent.text.replace('\\n', ' ').strip())\n",
    "    \n",
    "    return sents"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "7c735f10",
   "metadata": {},
   "outputs": [],
   "source": [
    "en_sents = sentencize(en_text)\n",
    "zh_sents = sentencize(zh_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "2218cb9b",
   "metadata": {},
   "outputs": [],
   "source": [
    "def align(en_sents, zh_sents):\n",
    "    embed = hub.load(\"https://tfhub.dev/google/universal-sentence-encoder-multilingual/3\")\n",
    "    en_result = embed(en_sents)\n",
    "    zh_result = embed(zh_sents)\n",
    "    sims = np.inner(en_result, zh_result)\n",
    "\n",
    "    costs = np.zeros((len(en_sents)+1, len(zh_sents)+1))\n",
    "    pointers = np.zeros((len(en_sents)+1, len(zh_sents)+1), dtype=int)\n",
    "\n",
    "    for i in range(1, len(en_sents)+1):\n",
    "        costs[i, 0] = costs[i-1, 0] + 1.\n",
    "    \n",
    "    for j in range(1, len(zh_sents)+1):\n",
    "        costs[0, j] = costs[0, j-1] + 1.\n",
    "\n",
    "    for i in range(1, len(en_sents)+1):\n",
    "        for j in range(1, len(zh_sents)+1):\n",
    "            choices = [\n",
    "                (costs[i-1, j-1] + (1. - sims[i-1, j-1]), 1),\n",
    "                (costs[i-1, j] + 1., 2),\n",
    "                (costs[i, j-1] + 1., 3)\n",
    "            ]\n",
    "            best_choice = sorted(choices, key=lambda x: x[0])[0]\n",
    "            costs[i, j], pointers[i, j] = best_choice\n",
    "\n",
    "    aligned = []\n",
    "    i, j = len(en_sents), len(zh_sents)\n",
    "    while i > 0 or j > 0:\n",
    "        if pointers[i, j] == 1:\n",
    "            i -= 1\n",
    "            j -= 1\n",
    "            aligned.append((en_sents[i], zh_sents[j]))\n",
    "        elif pointers[i, j] == 2:\n",
    "            i -= 1\n",
    "            aligned.append((en_sents[i], ''))\n",
    "        elif pointers[i, j] == 3:\n",
    "            j -= 1\n",
    "            aligned.append(('', zh_sents[j]))\n",
    "\n",
    "    aligned.reverse()\n",
    "\n",
    "    return aligned"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "bd8e6caf",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---\n",
      "categorical data Features having a discrete set of possible values.\n",
      "分类数据  一种特征，拥有一组离散的可能值。\n",
      "---\n",
      "Blahblah.\n",
      "\n",
      "---\n",
      "For example, consider a categorical feature named house style,  which has a discrete set of three possible values: Tudor, ranch,  colonial.\n",
      "以某个名为 house style 的分类特征为例，该特征拥有一组离散的可能值（共三个）， 即 Tudor, ranch, colonial。\n",
      "---\n",
      "By representing house style as categorical data, the model can learn the separate impacts of Tudor, ranch, and colonial on house price.\n",
      "通过将 house style 表示成分类数据，相应模型可以 学习 Tudor、ranch 和 colonial 分别对房价的影响。\n",
      "---\n",
      "Sometimes, values in the discrete set are mutually exclusive, and only one value can be applied to a given example.\n",
      "有时，离散集中的值是互斥的， 只能将其中一个值应用于指定样本。\n",
      "---\n",
      "For example, a car maker categorical feature would probably permit only a single value (Toyota) per example.\n",
      "例如，car maker 分类特征可能只允许一个样本有 一个值 (Toyota)。\n",
      "---\n",
      "\n",
      "你好。\n",
      "---\n",
      "Other times, more than one value may be applicable.\n",
      "在其他情况下，则可以应用多个值。\n",
      "---\n",
      "A single car could be painted more than one different color, so a car color categorical feature would likely permit a single example to have multiple values (for example, red and white).\n",
      "一辆车可能会被喷涂多种 不同的颜色，因此，car color 分类特征可能会允许单个样本具有多个值（例如 red 和 white）。\n",
      "---\n",
      "Categorical features are sometimes called discrete features.\n",
      "分类特征有时称为离散特征。\n",
      "---\n",
      "Contrast with numerical data\n",
      "与数值数据相对。\n",
      "---\n",
      "\n",
      "完\n"
     ]
    }
   ],
   "source": [
    "for en_sent, zh_sent in align(en_sents, zh_sents):\n",
    "    print('---')\n",
    "    print(en_sent)\n",
    "    print(zh_sent)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "31d5533a",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}