Character Encodings.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "f1d63046",
   "metadata": {},
   "outputs": [],
   "source": [
    "# modules we'll use\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "# helpful character encoding module\n",
    "import chardet\n",
    "\n",
    "# set seed for reproducibility\n",
    "np.random.seed(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "24915656",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "str"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# start with a string\n",
    "before = \"This is the euro symbol: €\"\n",
    "\n",
    "# check to see what datatype it is\n",
    "type(before)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "9699690e",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "after = before.encode(\"utf-8\", errors=\"replace\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "c8895b73",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "bytes"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(after)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "b56e6460",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "This is the euro symbol: €\n"
     ]
    }
   ],
   "source": [
    "print(after.decode(\"utf-8\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "5188fdcd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "This is the euro symbol: ?\n"
     ]
    }
   ],
   "source": [
    "# start with a string\n",
    "before = \"This is the euro symbol: €\"\n",
    "\n",
    "# encode it to a different encoding, replacing characters that raise errors\n",
    "after = before.encode(\"ascii\", errors = \"replace\")\n",
    "\n",
    "# convert it back to utf-8\n",
    "print(after.decode(\"ascii\"))\n",
    "\n",
    "# We've lost the original underlying byte string! It's been \n",
    "# replaced with the underlying byte string for the unknown character :("
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "a15e1c2e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'encoding': 'Windows-1252', 'confidence': 0.73, 'language': ''}\n"
     ]
    }
   ],
   "source": [
    "# look at the first ten thousand bytes to guess the character encoding\n",
    "with open(r'C:\\Users\\zargham abbas\\Documents\\Kaggle\\Data Cleaning\\ks-projects-201612.csv', 'rb') as rawdata:\n",
    "    result = chardet.detect(rawdata.read(10000))\n",
    "\n",
    "# check what the character encoding might be\n",
    "print(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "cc143a30",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\zargham abbas\\AppData\\Roaming\\Python\\Python38\\site-packages\\IPython\\core\\interactiveshell.py:3165: DtypeWarning: Columns (13,14,15) have mixed types.Specify dtype option on import or set low_memory=False.\n",
      "  has_raised = await self.run_ast_nodes(code_ast.body, cell_name,\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>name</th>\n",
       "      <th>category</th>\n",
       "      <th>main_category</th>\n",
       "      <th>currency</th>\n",
       "      <th>deadline</th>\n",
       "      <th>goal</th>\n",
       "      <th>launched</th>\n",
       "      <th>pledged</th>\n",
       "      <th>state</th>\n",
       "      <th>backers</th>\n",
       "      <th>country</th>\n",
       "      <th>usd pledged</th>\n",
       "      <th>Unnamed: 13</th>\n",
       "      <th>Unnamed: 14</th>\n",
       "      <th>Unnamed: 15</th>\n",
       "      <th>Unnamed: 16</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1000002330</td>\n",
       "      <td>The Songs of Adelaide &amp; Abullah</td>\n",
       "      <td>Poetry</td>\n",
       "      <td>Publishing</td>\n",
       "      <td>GBP</td>\n",
       "      <td>2015-10-09 11:36:00</td>\n",
       "      <td>1000</td>\n",
       "      <td>2015-08-11 12:12:28</td>\n",
       "      <td>0</td>\n",
       "      <td>failed</td>\n",
       "      <td>0</td>\n",
       "      <td>GB</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1000004038</td>\n",
       "      <td>Where is Hank?</td>\n",
       "      <td>Narrative Film</td>\n",
       "      <td>Film &amp; Video</td>\n",
       "      <td>USD</td>\n",
       "      <td>2013-02-26 00:20:50</td>\n",
       "      <td>45000</td>\n",
       "      <td>2013-01-12 00:20:50</td>\n",
       "      <td>220</td>\n",
       "      <td>failed</td>\n",
       "      <td>3</td>\n",
       "      <td>US</td>\n",
       "      <td>220</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1000007540</td>\n",
       "      <td>ToshiCapital Rekordz Needs Help to Complete Album</td>\n",
       "      <td>Music</td>\n",
       "      <td>Music</td>\n",
       "      <td>USD</td>\n",
       "      <td>2012-04-16 04:24:11</td>\n",
       "      <td>5000</td>\n",
       "      <td>2012-03-17 03:24:11</td>\n",
       "      <td>1</td>\n",
       "      <td>failed</td>\n",
       "      <td>1</td>\n",
       "      <td>US</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1000011046</td>\n",
       "      <td>Community Film Project: The Art of Neighborhoo...</td>\n",
       "      <td>Film &amp; Video</td>\n",
       "      <td>Film &amp; Video</td>\n",
       "      <td>USD</td>\n",
       "      <td>2015-08-29 01:00:00</td>\n",
       "      <td>19500</td>\n",
       "      <td>2015-07-04 08:35:03</td>\n",
       "      <td>1283</td>\n",
       "      <td>canceled</td>\n",
       "      <td>14</td>\n",
       "      <td>US</td>\n",
       "      <td>1283</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1000014025</td>\n",
       "      <td>Monarch Espresso Bar</td>\n",
       "      <td>Restaurants</td>\n",
       "      <td>Food</td>\n",
       "      <td>USD</td>\n",
       "      <td>2016-04-01 13:38:27</td>\n",
       "      <td>50000</td>\n",
       "      <td>2016-02-26 13:38:27</td>\n",
       "      <td>52375</td>\n",
       "      <td>successful</td>\n",
       "      <td>224</td>\n",
       "      <td>US</td>\n",
       "      <td>52375</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          ID                                               name   \\\n",
       "0  1000002330                    The Songs of Adelaide & Abullah   \n",
       "1  1000004038                                     Where is Hank?   \n",
       "2  1000007540  ToshiCapital Rekordz Needs Help to Complete Album   \n",
       "3  1000011046  Community Film Project: The Art of Neighborhoo...   \n",
       "4  1000014025                               Monarch Espresso Bar   \n",
       "\n",
       "        category  main_category  currency             deadline   goal   \\\n",
       "0          Poetry     Publishing       GBP  2015-10-09 11:36:00   1000   \n",
       "1  Narrative Film   Film & Video       USD  2013-02-26 00:20:50  45000   \n",
       "2           Music          Music       USD  2012-04-16 04:24:11   5000   \n",
       "3    Film & Video   Film & Video       USD  2015-08-29 01:00:00  19500   \n",
       "4     Restaurants           Food       USD  2016-04-01 13:38:27  50000   \n",
       "\n",
       "             launched  pledged       state  backers  country  usd pledged   \\\n",
       "0  2015-08-11 12:12:28        0      failed        0       GB            0   \n",
       "1  2013-01-12 00:20:50      220      failed        3       US          220   \n",
       "2  2012-03-17 03:24:11        1      failed        1       US            1   \n",
       "3  2015-07-04 08:35:03     1283    canceled       14       US         1283   \n",
       "4  2016-02-26 13:38:27    52375  successful      224       US        52375   \n",
       "\n",
       "  Unnamed: 13 Unnamed: 14 Unnamed: 15  Unnamed: 16  \n",
       "0         NaN         NaN         NaN          NaN  \n",
       "1         NaN         NaN         NaN          NaN  \n",
       "2         NaN         NaN         NaN          NaN  \n",
       "3         NaN         NaN         NaN          NaN  \n",
       "4         NaN         NaN         NaN          NaN  "
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# read in the file with the encoding detected by chardet\n",
    "kickstarter_2016 = pd.read_csv(r'C:\\Users\\zargham abbas\\Documents\\Kaggle\\Data Cleaning\\ks-projects-201612.csv', encoding='Windows-1252')\n",
    "\n",
    "# look at the first few lines\n",
    "kickstarter_2016.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "5b1ad94a",
   "metadata": {},
   "outputs": [],
   "source": [
    "kickstarter_2016.to_csv(\"ks-projects-201801-utf8.csv\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}