01_macula_data_overview.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "view-in-github"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/ryderwishart/biblical-machine-learning/blob/main/macula_data_overview.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "r7U1Y1KmCbhX"
      },
      "source": [
        "# MACULA Data Overview\n",
        "\n",
        "## Simple TSV Data Exploration Notebook\n",
        "\n",
        "A very soft intro to pulling down and processing macula-greek TSV data using the Pandas library. Load and explore column names, descriptions, and data values.\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "eAUCmr4eCbhY"
      },
      "source": [
        "## Setup\n",
        "First, import the necessary libraries."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "VimAfDy7CbhZ"
      },
      "outputs": [],
      "source": [
        "import pandas as pd\n",
        "import os\n",
        "import random"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xoV1THaMCbha"
      },
      "source": [
        "## Download and Load Data\n",
        "Here, download the TSV files using the `!wget` command and load them using pandas."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wYnoEff0Cbha"
      },
      "source": [
        "Choose which language data to load"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "bnrpkEumCbha"
      },
      "outputs": [],
      "source": [
        "language = \"Greek\"  # Change this to \"Hebrew\" to load Hebrew data - not yet implemented"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "uBDr_E58Cbha"
      },
      "outputs": [],
      "source": [
        "if language == \"Greek\":\n",
        "    if \"macula-greek.tsv\" not in [path for path in os.listdir()]:\n",
        "        !wget - O macula-greek.tsv https://raw.githubusercontent.com/Clear-Bible/macula-greek/main/Nestle1904/TSV/macula-greek.tsv\n",
        "    tsv_file = \"macula-greek.tsv\"\n",
        "elif language == \"Hebrew\":\n",
        "    if \"macula-hebrew.tsv\" not in [path for path in os.listdir()]:\n",
        "        !wget - O macula-hebrew.tsv < URL_FOR_HEBREW_TSV > # FIXME: need to use GIT LFS for this one\n",
        "    tsv_file = \"macula-hebrew.tsv\"\n",
        "else:\n",
        "    raise ValueError(\"Invalid language choice\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GfqI9RRnCbhb"
      },
      "source": [
        "Load the chosen TSV file"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "I_20dSk0Cbhb"
      },
      "outputs": [],
      "source": [
        "data = pd.read_csv(tsv_file, sep=\"\\t\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Rzz8ou68Cbhb"
      },
      "source": [
        "## Column Information\n",
        "The following sections describe each column header, the possible values, and their meanings."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "rNQwNfwhCbhb",
        "outputId": "c6ed3601-b730-4716-e66b-452e0810d5a8"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Note that empty or \"nan\" values occur when a column does not apply to a given word\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-0781946c-7e5c-43ae-8d8f-56aa300b6b22\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>column_name</th>\n",
              "      <th>num_unique_values</th>\n",
              "      <th>unique_values</th>\n",
              "      <th>description</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>xml:id</td>\n",
              "      <td>137779</td>\n",
              "      <td>[n40001001001, n40001001002, n40001001003, n40...</td>\n",
              "      <td>XML ids occur on every word and encode the cor...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>ref</td>\n",
              "      <td>137779</td>\n",
              "      <td>[MAT 1:1!1, MAT 1:1!2, MAT 1:1!3, MAT 1:1!4, M...</td>\n",
              "      <td>Verse!word reference to this edition of the Ne...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>role</td>\n",
              "      <td>10</td>\n",
              "      <td>[nan, s, v, vc, p, adv, o, aux, io, o2, apposi...</td>\n",
              "      <td>The clause-level role of the word.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>class</td>\n",
              "      <td>11</td>\n",
              "      <td>[noun, verb, det, conj, pron, prep, adj, adv, ...</td>\n",
              "      <td>On words, the class is the word's part of speech</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>type</td>\n",
              "      <td>9</td>\n",
              "      <td>[common, proper, nan, personal, relative, demo...</td>\n",
              "      <td>Indicates different types of pronominals</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>gloss</td>\n",
              "      <td>20021</td>\n",
              "      <td>[[The] book, of [the] genealogy, of Jesus, Chr...</td>\n",
              "      <td>SIL data, not Berean</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>text</td>\n",
              "      <td>19477</td>\n",
              "      <td>[Βίβλος, γενέσεως, Ἰησοῦ, Χριστοῦ, υἱοῦ, Δαυεὶ...</td>\n",
              "      <td>Text content associated with the ID</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>after</td>\n",
              "      <td>17</td>\n",
              "      <td>[ , ., ,, ·, ;, —, ὶ, ς, ε, χ, ὸ, ι, α, ί, ὁ, ...</td>\n",
              "      <td>Encodes the following character, including a b...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>lemma</td>\n",
              "      <td>5401</td>\n",
              "      <td>[βίβλος, γένεσις, Ἰησοῦς, Χριστός, υἱός, Δαυίδ...</td>\n",
              "      <td>Form of the word as it appears in a dictionary.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>normalized</td>\n",
              "      <td>18480</td>\n",
              "      <td>[Βίβλος, γενέσεως, Ἰησοῦ, Χριστοῦ, υἱοῦ, Δαυεί...</td>\n",
              "      <td>The normalized form of the token (i.e., no tra...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>strong</td>\n",
              "      <td>5339</td>\n",
              "      <td>[976, 1078, 2424, 5547, 5207, 1138, 11, 1080, ...</td>\n",
              "      <td>Strong's number for the lemma</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>morph</td>\n",
              "      <td>1055</td>\n",
              "      <td>[N-NSF, N-GSF, N-GSM, N-PRI, V-AAI-3S, T-ASM, ...</td>\n",
              "      <td>Morphological parsing codes</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>person</td>\n",
              "      <td>3</td>\n",
              "      <td>[nan, third, second, first]</td>\n",
              "      <td>Grammatical person</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>number</td>\n",
              "      <td>2</td>\n",
              "      <td>[singular, nan, plural]</td>\n",
              "      <td>Grammatical number</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>gender</td>\n",
              "      <td>3</td>\n",
              "      <td>[feminine, masculine, nan, neuter]</td>\n",
              "      <td>Grammatical gender values</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td>case</td>\n",
              "      <td>5</td>\n",
              "      <td>[nominative, genitive, nan, accusative, dative...</td>\n",
              "      <td>Grammatical case: nominative, genitive, dative...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16</th>\n",
              "      <td>tense</td>\n",
              "      <td>6</td>\n",
              "      <td>[nan, aorist, present, imperfect, future, perf...</td>\n",
              "      <td>Grammatical tense form</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17</th>\n",
              "      <td>voice</td>\n",
              "      <td>4</td>\n",
              "      <td>[nan, active, passive, middle, middlepassive]</td>\n",
              "      <td>Grammatical voice</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>18</th>\n",
              "      <td>mood</td>\n",
              "      <td>6</td>\n",
              "      <td>[nan, indicative, participle, infinitive, impe...</td>\n",
              "      <td>Grammatical mood</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>19</th>\n",
              "      <td>degree</td>\n",
              "      <td>2</td>\n",
              "      <td>[nan, superlative, comparative]</td>\n",
              "      <td>A derivative lexical category, indicating the ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>20</th>\n",
              "      <td>domain</td>\n",
              "      <td>1646</td>\n",
              "      <td>[033005, 010002 033003, 093001, 010002, 023003...</td>\n",
              "      <td>Semantic domain information from the Semantic ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>21</th>\n",
              "      <td>ln</td>\n",
              "      <td>7814</td>\n",
              "      <td>[33.38, 10.24 33.19, 93.169a, 93.387, 10.30, 9...</td>\n",
              "      <td>Short for Louw-Nida, representing the semantic...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>22</th>\n",
              "      <td>frame</td>\n",
              "      <td>20299</td>\n",
              "      <td>[nan, A0:n40001002001 A1:n40001002004, A0:n400...</td>\n",
              "      <td>Frames of verbs, refers to the arguments of th...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>23</th>\n",
              "      <td>subjref</td>\n",
              "      <td>5567</td>\n",
              "      <td>[nan, n40001016012, n40001018011, n40001019001...</td>\n",
              "      <td>The xml:id of the node that is the implied sub...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>24</th>\n",
              "      <td>referent</td>\n",
              "      <td>6099</td>\n",
              "      <td>[nan, n40001002014, n40001011005, n40001016008...</td>\n",
              "      <td>The xml:id of the node to which a pronoun (i.e...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-0781946c-7e5c-43ae-8d8f-56aa300b6b22')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-0781946c-7e5c-43ae-8d8f-56aa300b6b22 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-0781946c-7e5c-43ae-8d8f-56aa300b6b22');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "   column_name num_unique_values  \\\n",
              "0       xml:id            137779   \n",
              "1          ref            137779   \n",
              "2         role                10   \n",
              "3        class                11   \n",
              "4         type                 9   \n",
              "5        gloss             20021   \n",
              "6         text             19477   \n",
              "7        after                17   \n",
              "8        lemma              5401   \n",
              "9   normalized             18480   \n",
              "10      strong              5339   \n",
              "11       morph              1055   \n",
              "12      person                 3   \n",
              "13      number                 2   \n",
              "14      gender                 3   \n",
              "15        case                 5   \n",
              "16       tense                 6   \n",
              "17       voice                 4   \n",
              "18        mood                 6   \n",
              "19      degree                 2   \n",
              "20      domain              1646   \n",
              "21          ln              7814   \n",
              "22       frame             20299   \n",
              "23     subjref              5567   \n",
              "24    referent              6099   \n",
              "\n",
              "                                        unique_values  \\\n",
              "0   [n40001001001, n40001001002, n40001001003, n40...   \n",
              "1   [MAT 1:1!1, MAT 1:1!2, MAT 1:1!3, MAT 1:1!4, M...   \n",
              "2   [nan, s, v, vc, p, adv, o, aux, io, o2, apposi...   \n",
              "3   [noun, verb, det, conj, pron, prep, adj, adv, ...   \n",
              "4   [common, proper, nan, personal, relative, demo...   \n",
              "5   [[The] book, of [the] genealogy, of Jesus, Chr...   \n",
              "6   [Βίβλος, γενέσεως, Ἰησοῦ, Χριστοῦ, υἱοῦ, Δαυεὶ...   \n",
              "7   [ , ., ,, ·, ;, —, ὶ, ς, ε, χ, ὸ, ι, α, ί, ὁ, ...   \n",
              "8   [βίβλος, γένεσις, Ἰησοῦς, Χριστός, υἱός, Δαυίδ...   \n",
              "9   [Βίβλος, γενέσεως, Ἰησοῦ, Χριστοῦ, υἱοῦ, Δαυεί...   \n",
              "10  [976, 1078, 2424, 5547, 5207, 1138, 11, 1080, ...   \n",
              "11  [N-NSF, N-GSF, N-GSM, N-PRI, V-AAI-3S, T-ASM, ...   \n",
              "12                        [nan, third, second, first]   \n",
              "13                            [singular, nan, plural]   \n",
              "14                 [feminine, masculine, nan, neuter]   \n",
              "15  [nominative, genitive, nan, accusative, dative...   \n",
              "16  [nan, aorist, present, imperfect, future, perf...   \n",
              "17      [nan, active, passive, middle, middlepassive]   \n",
              "18  [nan, indicative, participle, infinitive, impe...   \n",
              "19                    [nan, superlative, comparative]   \n",
              "20  [033005, 010002 033003, 093001, 010002, 023003...   \n",
              "21  [33.38, 10.24 33.19, 93.169a, 93.387, 10.30, 9...   \n",
              "22  [nan, A0:n40001002001 A1:n40001002004, A0:n400...   \n",
              "23  [nan, n40001016012, n40001018011, n40001019001...   \n",
              "24  [nan, n40001002014, n40001011005, n40001016008...   \n",
              "\n",
              "                                          description  \n",
              "0   XML ids occur on every word and encode the cor...  \n",
              "1   Verse!word reference to this edition of the Ne...  \n",
              "2                  The clause-level role of the word.  \n",
              "3    On words, the class is the word's part of speech  \n",
              "4            Indicates different types of pronominals  \n",
              "5                                SIL data, not Berean  \n",
              "6                 Text content associated with the ID  \n",
              "7   Encodes the following character, including a b...  \n",
              "8     Form of the word as it appears in a dictionary.  \n",
              "9   The normalized form of the token (i.e., no tra...  \n",
              "10                      Strong's number for the lemma  \n",
              "11                        Morphological parsing codes  \n",
              "12                                 Grammatical person  \n",
              "13                                 Grammatical number  \n",
              "14                          Grammatical gender values  \n",
              "15  Grammatical case: nominative, genitive, dative...  \n",
              "16                             Grammatical tense form  \n",
              "17                                  Grammatical voice  \n",
              "18                                   Grammatical mood  \n",
              "19  A derivative lexical category, indicating the ...  \n",
              "20  Semantic domain information from the Semantic ...  \n",
              "21  Short for Louw-Nida, representing the semantic...  \n",
              "22  Frames of verbs, refers to the arguments of th...  \n",
              "23  The xml:id of the node that is the implied sub...  \n",
              "24  The xml:id of the node to which a pronoun (i.e...  "
            ]
          },
          "execution_count": 38,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# Create an empty list to store dictionaries for each column\n",
        "column_info_list = []\n",
        "\n",
        "# Iterate through each column in the data DataFrame\n",
        "for column in data.columns:\n",
        "    num_unique_values = data[column].nunique()\n",
        "    unique_values = data[column].unique()\n",
        "    column_info = {\"column_name\": column,\n",
        "                   \"num_unique_values\": num_unique_values,\n",
        "                   \"unique_values\": unique_values}\n",
        "    column_info_list.append(column_info)\n",
        "\n",
        "# Convert the list of dictionaries into a DataFrame\n",
        "overview = pd.concat([pd.Series(d) for d in column_info_list], axis=1).T\n",
        "\n",
        "# Create a dictionary with attribute descriptions\n",
        "attribute_descriptions = {\n",
        "    \"after\": \"Encodes the following character, including a blank space.\",\n",
        "    \"articular\": \"'true' if the word has an article (i.e., modified by the word 'the').\",\n",
        "    \"case\": \"Grammatical case: nominative, genitive, dative, accusative, or vocative\",\n",
        "    \"class\": \"On words, the class is the word's part of speech\",\n",
        "    \"cltype\": \"Explicitly marks Verbless Clauses, Verb Elided Clauses, and Minor Clauses\",\n",
        "    \"degree\": \"A derivative lexical category, indicating the degree of the adjective\",\n",
        "    \"discontinuous\": \"'true' if the word is discontinuous with respect to sentence order due to reordering in the syntax tree\",\n",
        "    \"domain\": \"Semantic domain information from the Semantic Dictionary of Biblical Greek (SDBG)\",\n",
        "    \"frame\": \"Frames of verbs, refers to the arguments of the verb\",\n",
        "    \"gender\": \"Grammatical gender values\",\n",
        "    \"gloss\": \"SIL data, not Berean\",\n",
        "    \"lemma\": \"Form of the word as it appears in a dictionary.\",\n",
        "    \"ln\": \"Short for Louw-Nida, representing the semantic domain entry in Johannes P. Louw and Eugene Albert Nida, Greek-English Lexicon of the New Testament: Based on Semantic Domains (New York: United Bible Societies, 1996).\",\n",
        "    \"mood\": \"Grammatical mood\",\n",
        "    \"morph\": \"Morphological parsing codes\",\n",
        "    \"normalized\": \"The normalized form of the token (i.e., no trailing or leading punctuation or accent shifting depending on context)\",\n",
        "    \"number\": \"Grammatical number\",\n",
        "    \"person\": \"Grammatical person\",\n",
        "    \"ref\": \"Verse!word reference to this edition of the Nestle1904 text by USFM id\",\n",
        "    \"referent\": \"The xml:id of the node to which a pronoun (i.e., 'he') refers. Note that some of these IDs are not word IDs but rather phrase or clause IDs.\",\n",
        "    \"role\": \"The clause-level role of the word.\",\n",
        "    \"strong\": \"Strong's number for the lemma\",\n",
        "    \"subjref\": \"The xml:id of the node that is the implied subject of a verb (for verbs without an explicit subject). Note that some of these IDs are not word IDs but rather phrase or clause IDs.\",\n",
        "    \"tense\": \"Grammatical tense form\",\n",
        "    \"text\": \"Text content associated with the ID\",\n",
        "    \"type\": \"Indicates different types of pronominals\",\n",
        "    \"voice\": \"Grammatical voice\",\n",
        "    \"xml:id\": \"XML ids occur on every word and encode the corpus ('n' for New Testament), the book (40 for Matthew), the chapter (001), verse (001), and word (001).\"\n",
        "}\n",
        "\n",
        "# Add descriptions to the overview dataframe\n",
        "overview[\"description\"] = overview[\"column_name\"].map(attribute_descriptions)\n",
        "\n",
        "# Display the overview dataframe with descriptions\n",
        "print('Note that empty or \"nan\" values occur when a column does not apply to a given word')\n",
        "overview\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "eyi4VYmtUPM2"
      },
      "source": [
        "## Inspect a column\n",
        "\n",
        "Specify which column name you would like to look at some examples of, and then view five random rows from the dataset as examples."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "FwpRZ70kUYKW",
        "outputId": "f42a11c5-29a8-46b1-f7f8-64b05d105c53"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "COLUMN NAME: mood\n",
            "COLUMN DESCRIPTION: Grammatical mood\n",
            "\n",
            "Five (or fewer) random non-null cells from the 'mood' and 'text' columns:\n",
            "              mood        text\n",
            "28035   participle       ἐλθὼν\n",
            "66956   indicative  ἐμελέτησαν\n",
            "29496  subjunctive      πίωσιν\n",
            "44827   participle   ἐρχόμενος\n",
            "4732    indicative  ἠνεῴχθησαν\n"
          ]
        }
      ],
      "source": [
        "# Specify the column name in a variable\n",
        "column_name = \"mood\"  # Replace with your desired column name\n",
        "\n",
        "# Filter out null values from the specified column\n",
        "non_null_data = data[data[column_name].notnull()]\n",
        "\n",
        "# If there are fewer than 5 non-null rows, adjust the sample size accordingly\n",
        "num_non_null_rows = len(non_null_data)\n",
        "sample_size = min(5, num_non_null_rows)\n",
        "\n",
        "# Select 5 (or fewer) random non-null rows\n",
        "random_rows = random.sample(range(num_non_null_rows), sample_size)\n",
        "random_cells = non_null_data[[column_name, 'text']].iloc[random_rows]\n",
        "\n",
        "# Get description from overview dataset\n",
        "column_description = overview[overview['column_name'] == column_name]['description'].values[0]\n",
        "\n",
        "print(f\"COLUMN NAME: {column_name}\\nCOLUMN DESCRIPTION: {column_description}\\n\")\n",
        "print(f\"Five (or fewer) random non-null cells from the '{column_name}' and 'text' columns:\")\n",
        "print(random_cells)\n"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "include_colab_link": true,
      "provenance": [],
      "toc_visible": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.6.4"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}