Text Analysis in Python and nltk. Week 1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Splitting of a text into words #"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['The', 'cornet', 'is', 'like', 'the', 'trumpet,', 'but', 'has', 'a', 'conical', 'bore,', 'more', 'compact', 'shape,', 'and', 'mellow', 'tone']\n"
     ]
    }
   ],
   "source": [
    "text1='The cornet is like the trumpet, but has a conical bore, more compact shape, and mellow tone'\n",
    "text2=text1.split(' ')\n",
    "print(text2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Using endswith(c), startswith(c) #"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['The', 'like', 'the', 'more', 'tone']\n"
     ]
    }
   ],
   "source": [
    "text3=[item for item in text2 if item.endswith('e')]\n",
    "print(text3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Using isupper(), islower(), istitle() #"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['cornet', 'is', 'like', 'the', 'trumpet,', 'but', 'has', 'a', 'conical', 'bore,', 'more', 'compact', 'shape,', 'and', 'mellow', 'tone']\n"
     ]
    }
   ],
   "source": [
    "text4=[item for item in text2 if item.islower()]\n",
    "print(text4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Using lower(), upper(), title() #"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['THE', 'CORNET', 'IS', 'LIKE', 'THE', 'TRUMPET,', 'BUT', 'HAS', 'A', 'CONICAL', 'BORE,', 'MORE', 'COMPACT', 'SHAPE,', 'AND', 'MELLOW', 'TONE']\n"
     ]
    }
   ],
   "source": [
    "text5=[item.upper() for item in text2]\n",
    "print(text5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Number of words #"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "17\n"
     ]
    }
   ],
   "source": [
    "print(len(text2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Joining a list about a string #"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The cornet is like the trumpet, but has a conical bore, more compact shape, and mellow tone\n"
     ]
    }
   ],
   "source": [
    "text6=' '.join(text2)\n",
    "print(text6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Functions applicable on text directly ##"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Using strip() (from both start and the end) or rstrip() (from end only) #  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  The cornet is like the trumpet, but has a conical bore, more compact shape, and mellow tone  \n",
      "The cornet is like the trumpet, but has a conical bore, more compact shape, and mellow tone\n"
     ]
    }
   ],
   "source": [
    "text1='  The cornet is like the trumpet, but has a conical bore, more compact shape, and mellow tone  '\n",
    "text2=text1.strip()\n",
    "print(text1)\n",
    "print(text2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# split() #"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['The', 'cornet', 'is', 'like', 'the', 'trumpet,', 'but', 'has', 'a', 'conical', 'bore,', 'more', 'compact', 'shape,', 'and', 'mellow', 'tone']\n"
     ]
    }
   ],
   "source": [
    "text3=text2.split()\n",
    "print(text3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "# lower(), upper(), title() #"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "THE CORNET IS LIKE THE TRUMPET, BUT HAS A CONICAL BORE, MORE COMPACT SHAPE, AND MELLOW TONE\n"
     ]
    }
   ],
   "source": [
    "text4=text2.upper()\n",
    "print(text4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "# replace() # "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "THE CORNET IS LIKE THE TRUMPET, BUT HxxS xx CONICxxL BORE, MORE COMPxxCT SHxxPE, xxND MELLOW TONE\n"
     ]
    }
   ],
   "source": [
    "text5=text4.replace('A', 'xx')\n",
    "print(text5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Searching for a substring #"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "True\n"
     ]
    }
   ],
   "source": [
    "val='NET' in text5\n",
    "print(val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4\n"
     ]
    }
   ],
   "source": [
    "print(text5.find('CO'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "64\n"
     ]
    }
   ],
   "source": [
    "print(text5.rfind('CO'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "# splitlines() (to split about \\n) #"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['The cornet is like the trumpet, but has a conical bore, more compact shape, and mellow tone']\n"
     ]
    }
   ],
   "source": [
    "text6=text2.splitlines()\n",
    "print(text6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Reading a file into text #"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "To ,\n",
      "The General Secretary\n",
      "Gopali Youth Welfare Society\n",
      "No-Shooting Area, Salua\n",
      "Date: 3rd July 2019\n",
      "Subject: Regarding payment of T-shirts of Helping Hands.\n",
      "Sir,\n",
      "Few days ago we ordered T-shirts for students of schools who actively participated in Helping Hands 2018 and\n",
      "contributed a big donation. This is to inform you that a sum of INR 14,500 + 5% GST i.e. total of INR 15,225 is\n",
      "needed. The T-Shirts will be directly send to the schools and the above mentioned sum includes Courier Charges\n",
      "also.\n",
      "We will be paying the above mentioned sum in two installations one of 10150 i.e. 70% then the rest of the money\n",
      "and GST when T-Shirts will get printed.\n",
      "Kindly allow me to use the above mentioned sum for this purpose.\n",
      "Details of the Dealer:\n",
      "Co. Name - Incorporated Pride\n",
      "Name - Victor\n",
      "204, 24c Rajanigandha\n",
      "Hiranandani Garden, Powai\n",
      "Mumbai- 400076\n",
      "Landmark- Nr. S.M Shetty School\n",
      "GST No.- 27BAYPS6974A1ZX\n",
      "Your Faithfully\n",
      "Saurav Kumar Nishant\n",
      "Finance Head, GYWS\n"
     ]
    }
   ],
   "source": [
    "f=open('Letter.txt', 'r')\n",
    "f.seek(0)\n",
    "text=f.read()\n",
    "print(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['To ,', 'The General Secretary', 'Gopali Youth Welfare Society', 'No-Shooting Area, Salua', 'Date: 3rd July 2019', 'Subject: Regarding payment of T-shirts of Helping Hands.', 'Sir,', 'Few days ago we ordered T-shirts for students of schools who actively participated in Helping Hands 2018 and', 'contributed a big donation. This is to inform you that a sum of INR 14,500 + 5% GST i.e. total of INR 15,225 is', 'needed. The T-Shirts will be directly send to the schools and the above mentioned sum includes Courier Charges', 'also.', 'We will be paying the above mentioned sum in two installations one of 10150 i.e. 70% then the rest of the money', 'and GST when T-Shirts will get printed.', 'Kindly allow me to use the above mentioned sum for this purpose.', 'Details of the Dealer:', 'Co. Name - Incorporated Pride', 'Name - Victor', '204, 24c Rajanigandha', 'Hiranandani Garden, Powai', 'Mumbai- 400076', 'Landmark- Nr. S.M Shetty School', 'GST No.- 27BAYPS6974A1ZX', 'Your Faithfully', 'Saurav Kumar Nishant', 'Finance Head, GYWS']\n"
     ]
    }
   ],
   "source": [
    "text1=text.splitlines()\n",
    "print(text1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Regular expressions #"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "# We can use all text operarions with DataFrame using 'str' #"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Saurav</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Hi</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Nice one</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>He loves coding</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              text\n",
       "0           Saurav\n",
       "1               Hi\n",
       "2         Nice one\n",
       "3  He loves coding"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val=pd.DataFrame({'text':['Saurav', 'Hi', 'Nice one', 'He loves coding']})\n",
    "val"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0     6\n",
       "1     2\n",
       "2     8\n",
       "3    15\n",
       "Name: text, dtype: int64"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "temp=val['text'].str.len()\n",
    "temp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0             SAURAV\n",
       "1                 HI\n",
       "2           NICE ONE\n",
       "3    HE LOVES CODING\n",
       "Name: text, dtype: object"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "temp=val['text'].str.upper()\n",
    "temp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0                 Saurav\n",
       "1                     Hi\n",
       "2           Nicxxx onxxx\n",
       "3    Hxxx lovxxxs coding\n",
       "Name: text, dtype: object"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "temp=val['text'].str.replace('e', 'xxx')\n",
    "temp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0                   SAURAV\n",
       "1                       HI\n",
       "2           NICxxxx ONxxxx\n",
       "3    Hxxxx LOVxxxxS CODING\n",
       "Name: text, dtype: object"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "temp=val['text'].str.upper().str.replace('E', 'xxxx')\n",
    "temp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "## ASCII encoding and UTF-8 encoding ##"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ASCII has only 128 values that includes English alphabets, numbers(0-9), some characters .\n",
    "# But UTF-8 provides a much bigger platform and provides many other languages, emoji etc which is not possible in Ascii\n",
    "# Now UTF-8 is widely accepted encoding scheme than ASCII "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}