custom2.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "katse_kaks.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "p410xmDocDKF",
        "outputId": "f1346da8-0be3-49e8-dae8-b8d91b75bff7"
      },
      "source": [
        "import pandas as pd\r\n",
        "import numpy as np\r\n",
        "from keras.utils import np_utils\r\n",
        "import re\r\n",
        "import sys \r\n",
        "from keras.models import Sequential\r\n",
        "from keras.layers import LSTM, Activation, Flatten, Dropout, Dense, Embedding, TimeDistributed, LSTM\r\n",
        "from keras.callbacks import LambdaCallback, ModelCheckpoint\r\n",
        "import tensorflow as tf\r\n",
        "from keras.optimizers import RMSprop, Adam\r\n",
        "import random\r\n",
        "import os\r\n",
        "!pip install pyarrow==2.0.0"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting pyarrow==2.0.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/d7/e1/27958a70848f8f7089bff8d6ebe42519daf01f976d28b481e1bfd52c8097/pyarrow-2.0.0-cp36-cp36m-manylinux2014_x86_64.whl (17.7MB)\n",
            "\u001b[K     |████████████████████████████████| 17.7MB 208kB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy>=1.14 in /usr/local/lib/python3.6/dist-packages (from pyarrow==2.0.0) (1.19.4)\n",
            "Installing collected packages: pyarrow\n",
            "  Found existing installation: pyarrow 0.14.1\n",
            "    Uninstalling pyarrow-0.14.1:\n",
            "      Successfully uninstalled pyarrow-0.14.1\n",
            "Successfully installed pyarrow-2.0.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6ExbkDqWcF9G"
      },
      "source": [
        "import requests\r\n",
        "import io\r\n",
        "data = pd.read_feather('https://github.com/Scytheface/NARV/raw/main/lyrics/lyrics.ft')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PEDZfRsEcKxe"
      },
      "source": [
        "NoneType = type(None) \r\n",
        "text = \"\"\r\n",
        "for index, row in data.iterrows():\r\n",
        "  if isinstance(row['lyrics'], NoneType):\r\n",
        "    continue\r\n",
        "  else:\r\n",
        "    text += row['lyrics'].lower()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IiPlE2Fkcd9_"
      },
      "source": [
        "chars = sorted(list(set(text)))\r\n",
        "char_indices = dict((c, i) for i, c in enumerate(chars))\r\n",
        "indices_char = dict((i, c) for i, c in enumerate(chars))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UjqibI9-cf3R",
        "outputId": "7fd0b8f4-fe31-4a3c-db76-e862cd19c5e3"
      },
      "source": [
        "maxlen = 50\r\n",
        "step = 3\r\n",
        "sentences = []\r\n",
        "next_chars = []\r\n",
        "for i in range(0, len(text) - maxlen, step):\r\n",
        "    sentences.append(text[i: i + maxlen])\r\n",
        "    next_chars.append(text[i + maxlen])\r\n",
        "print('Number of Sequences:', len(sentences))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Number of Sequences: 594117\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Xku4_XQ9cheu"
      },
      "source": [
        "x = np.zeros((len(sentences), maxlen, len(chars)), dtype=np.bool)\r\n",
        "y = np.zeros((len(sentences), len(chars)), dtype=np.bool)\r\n",
        "for i, sentence in enumerate(sentences):\r\n",
        "    for t, char in enumerate(sentence):\r\n",
        "        x[i, t, char_indices[char]] = 1\r\n",
        "    y[i, char_indices[next_chars[i]]] = 1"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1oo6oKyrcjf1"
      },
      "source": [
        "model = Sequential()\r\n",
        "model.add(LSTM(64, input_shape=(maxlen, len(chars)), return_sequences=True))\r\n",
        "model.add(Dropout(0.2))\r\n",
        "model.add(LSTM(64, input_shape=(maxlen, len(chars)), return_sequences=True))\r\n",
        "model.add(Dropout(0.2))\r\n",
        "model.add(LSTM(64, input_shape=(maxlen, len(chars)), return_sequences=True))\r\n",
        "model.add(Dropout(0.2))\r\n",
        "model.add(LSTM(64, input_shape=(maxlen, len(chars)), return_sequences=True))\r\n",
        "model.add(Dropout(0.2))\r\n",
        "model.add(LSTM(64, input_shape=(maxlen, len(chars))))\r\n",
        "model.add(Dropout(0.2))\r\n",
        "model.add(Dense(len(chars)))\r\n",
        "model.add(Activation('softmax'))\r\n",
        "\r\n",
        "optimizer = RMSprop(lr=0.001, momentum=0.1)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EBedE9QTkf1f",
        "outputId": "de8b8a16-3950-4b9b-a271-cc7d857bb837"
      },
      "source": [
        "#weights_file = '/content/lyrics-weights.hdf5'\r\n",
        "#model.load_weights(weights_file)\r\n",
        "model.compile(loss='categorical_crossentropy', optimizer=optimizer)\r\n",
        "model.summary()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"sequential_1\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "lstm_5 (LSTM)                (None, 50, 64)            51200     \n",
            "_________________________________________________________________\n",
            "dropout_5 (Dropout)          (None, 50, 64)            0         \n",
            "_________________________________________________________________\n",
            "lstm_6 (LSTM)                (None, 50, 64)            33024     \n",
            "_________________________________________________________________\n",
            "dropout_6 (Dropout)          (None, 50, 64)            0         \n",
            "_________________________________________________________________\n",
            "lstm_7 (LSTM)                (None, 50, 64)            33024     \n",
            "_________________________________________________________________\n",
            "dropout_7 (Dropout)          (None, 50, 64)            0         \n",
            "_________________________________________________________________\n",
            "lstm_8 (LSTM)                (None, 50, 64)            33024     \n",
            "_________________________________________________________________\n",
            "dropout_8 (Dropout)          (None, 50, 64)            0         \n",
            "_________________________________________________________________\n",
            "lstm_9 (LSTM)                (None, 64)                33024     \n",
            "_________________________________________________________________\n",
            "dropout_9 (Dropout)          (None, 64)                0         \n",
            "_________________________________________________________________\n",
            "dense_1 (Dense)              (None, 135)               8775      \n",
            "_________________________________________________________________\n",
            "activation_1 (Activation)    (None, 135)               0         \n",
            "=================================================================\n",
            "Total params: 192,071\n",
            "Trainable params: 192,071\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Qtxb6-Rid4h4"
      },
      "source": [
        "filepath = \"lyrics-weights.hdf5\"\r\n",
        "checkpoint = ModelCheckpoint(filepath, \r\n",
        "                             monitor='val_loss', \r\n",
        "                             verbose=1, \r\n",
        "                             save_best_only=True, \r\n",
        "                             mode='min')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xDU7Vm10cnPr"
      },
      "source": [
        "def sample(preds, diversity):\r\n",
        "    preds = np.asarray(preds).astype('float64')\r\n",
        "    preds = np.log(preds) / diversity\r\n",
        "    exp_preds = np.exp(preds)\r\n",
        "    preds = exp_preds / np.sum(exp_preds)\r\n",
        "    probas = np.random.multinomial(1, preds, 1)\r\n",
        "    return np.argmax(probas)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0hgq0lfRcq6v",
        "outputId": "0992d190-aa3c-47b4-b43c-8a90dd269306"
      },
      "source": [
        "model.fit(x, y,\r\n",
        "          batch_size=64,\r\n",
        "          epochs=25,\r\n",
        "          validation_split = 0.04,\r\n",
        "          verbose=1,\r\n",
        "          callbacks=[checkpoint])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/25\n",
            "2777/8912 [========>.....................] - ETA: 1:36 - loss: 3.1410"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yNAM2HayctVM"
      },
      "source": [
        "weights_file = '/content/lyrics-weights.hdf5'\r\n",
        "model.load_weights(weights_file)\r\n",
        "model.compile(loss = 'categorical_crossentropy', optimizer = 'rmsprop')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "A6g1CyosvA16"
      },
      "source": [
        "start_index = random.randint(0, len(text) - maxlen - 1)\r\n",
        "generated = ''\r\n",
        "sentence = text[start_index: start_index + maxlen]\r\n",
        "generated += sentence\r\n",
        "print('***** Generating with: \"' + sentence + '\"')\r\n",
        "sys.stdout.write(generated)\r\n",
        "\r\n",
        "for i in range(500):\r\n",
        "    x_pred = np.zeros((1, maxlen, len(chars)))\r\n",
        "    for t, char in enumerate(sentence):\r\n",
        "        x_pred[0, t, char_indices[char]] = 1.\r\n",
        "\r\n",
        "    preds = model.predict(x_pred, verbose=0)[0]\r\n",
        "    next_index = sample(preds, 0.5)\r\n",
        "    next_char = indices_char[next_index]\r\n",
        "\r\n",
        "    generated += next_char\r\n",
        "    sentence = sentence[1:] + next_char\r\n",
        "\r\n",
        "    sys.stdout.write(next_char)\r\n",
        "    sys.stdout.flush()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ppc7SSIjvPsX"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}