Web-Scraping + Salary Classification Prediction.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 262,
   "metadata": {
    "collapsed": false,
    "focus": false,
    "id": "78446809-fa02-48df-b60f-cbeda175a498"
   },
   "outputs": [],
   "source": [
    "import requests\n",
    "import bs4\n",
    "from bs4 import BeautifulSoup\n",
    "import pandas as pd\n",
    "from sklearn import linear_model, model_selection, tree, ensemble, svm, \\\n",
    "metrics, feature_extraction, neighbors, naive_bayes, feature_selection\n",
    "import xgboost\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import math\n",
    "import numpy as np\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Web Scraping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "cities = ['Atlanta, GA', 'Washington, DC', 'New York, NY', 'New Orleans, LA', 'Boston, MA', \n",
    "         'Austin, TX', 'Seattle, WA', 'San Francisco, CA', 'Detroit, MI', 'Minneapolis, MN',]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false,
    "focus": false,
    "id": "b02e2931-4d5a-4e1e-9504-c6ccaaf84bed"
   },
   "outputs": [],
   "source": [
    "# Template URL\n",
    "url_template = \"http://www.indeed.com/jobs?q=data+scientist&l={}&start={}\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# for loop to pull data with bs4\n",
    "results = []\n",
    "starts = [] #I want to keep track of what page something appears on\n",
    "city_state = [] #I want to keep track of what city I searched in, since not all results are in the city.\n",
    "for city in cities:\n",
    "    city = city.replace(' ', '+')\n",
    "    for start in range(0,2000,10):\n",
    "        r = requests.get(url_template.format(city, start))\n",
    "        # Grab the results from the request (as above)\n",
    "        soup = BeautifulSoup(r.content, 'lxml')\n",
    "        # Append to the full set of results\n",
    "        results += soup.findAll('div', { \"class\" : \"result\" })\n",
    "        starts.extend([start]*len(soup.findAll('div', { \"class\" : \"result\" })))\n",
    "        city_state.extend([city]*len(soup.findAll('div', { \"class\" : \"result\" })))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Extraction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def extract_text(el):\n",
    "    if el:\n",
    "        return el.text.strip()\n",
    "    else:\n",
    "        return ''\n",
    "extract_text(results[1])\n",
    "\n",
    "# company\n",
    "def get_company_from_result(result):\n",
    "    return extract_text(result.find('span', {'class' : 'company'}))\n",
    "\n",
    "# location\n",
    "def get_location_from_result(result):\n",
    "    return extract_text(result.find('span', {'class':'location'}))\n",
    "\n",
    "# summary\n",
    "def get_summary_from_result(result):\n",
    "    return extract_text(result.find('span', {'class':'summary'}))\n",
    "\n",
    "# title\n",
    "def get_title_from_result(result):\n",
    "    return extract_text(result.find('a', {'data-tn-element' : 'jobTitle'}))\n",
    "\n",
    "# get salary if exists\n",
    "def get_salary_from_result(result):\n",
    "    salary_table = result.find('td', {'class' : 'snip'})\n",
    "    if salary_table:\n",
    "        snip = salary_table.find('nobr')\n",
    "        if snip:\n",
    "            return snip.text.strip()   \n",
    "    return None\n",
    "\n",
    "def get_date_posted(result):\n",
    "    return extract_text(result.find('span', {'class':'date'}))\n",
    "\n",
    "# whether sponsored may be associated with salary (sponsored jobs may indicate a company with more money, or more desperate?)\n",
    "def get_is_sponsored(result):\n",
    "    is_sponsored = extract_text(result.find('span', {'class':'sdn'}))\n",
    "    if is_sponsored:\n",
    "        return is_sponsored\n",
    "    return None\n",
    "\n",
    "# number of reviews may be associated with salary\n",
    "def get_number_reviews(result):\n",
    "    x = extract_text(result.find('span', {'class': 'slNoUnderline'}))\n",
    "    if x:\n",
    "        return x\n",
    "    return None\n",
    "\n",
    "# get star rating\n",
    "import re\n",
    "\n",
    "def get_star_rating(result):\n",
    "    find_rating = result.find('span',{'class':'rating'})\n",
    "    if find_rating:\n",
    "        search_in = str(find_rating)\n",
    "        return 5*float(re.findall('width: (.*)px', search_in)[0])/60\n",
    "    return None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {
    "collapsed": false,
    "focus": false,
    "id": "d601ff2f-fbdf-4c4f-8bbe-10c4a3132cc8"
   },
   "outputs": [],
   "source": [
    "# combine data into dictionaries company location summary title salary\n",
    "rows = []\n",
    "for i, result in enumerate(results):\n",
    "    if result:\n",
    "        row = {'company': get_company_from_result(result),\n",
    "              'location': get_location_from_result(result),\n",
    "              'summary': get_summary_from_result(result),\n",
    "              'title': get_title_from_result(result),\n",
    "              'salary': get_salary_from_result(result),\n",
    "              'date_posted': get_date_posted(result),\n",
    "              'sponsored': get_is_sponsored(result),\n",
    "              'star_rating': get_star_rating(result),\n",
    "              'search_city':city_state[i],\n",
    "              'start': starts[i],\n",
    "              'number_reviews': get_number_reviews(result)}\n",
    "        rows.append(row)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# create dataframe\n",
    "ds_jobs = pd.DataFrame(rows)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Feature Extraction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# making a copy so I can refer to original if needed without having to re-run\n",
    "ds_jobs_clean = ds_jobs.copy()\n",
    "\n",
    "# drop duplicates (not considering 'start' column because I want to eliminate things that appeared more than once)\n",
    "ds_jobs_clean.drop_duplicates([x for x in ds_jobs.columns if x != 'start'], inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Extracting yearly salary from various data formats\n",
    "import re\n",
    "import numpy as np\n",
    "def get_standardized_salary(salary_string):\n",
    "    if salary_string:\n",
    "        if re.findall('(.*) a year', salary_string):\n",
    "            matches = re.findall('([0-9]+,[0-9]+)', salary_string)\n",
    "            return np.mean([float(salary.replace(',', '')) for salary in matches ])\n",
    "        elif re.findall('(.*) a month', salary_string):\n",
    "            matches = re.findall('([0-9]+,[0-9]+|[0-9]+)', salary_string)\n",
    "            return np.mean([float(salary.replace(',', '')) for salary in matches ])*12\n",
    "        elif re.findall('(.*) a week', salary_string):\n",
    "            matches = re.findall('([0-9]+,[0-9]+|[0-9]+)', salary_string)\n",
    "            return np.mean([float(salary.replace(',', '')) for salary in matches ])*52\n",
    "        elif re.findall('(.*) a day', salary_string):\n",
    "            matches = re.findall('([0-9]+,[0-9]+|[0-9]+)', salary_string)\n",
    "            return np.mean([float(salary.replace(',', '')) for salary in matches ])*5*52\n",
    "        elif re.findall('(.*) an hour', salary_string):\n",
    "            matches = re.findall('([0-9]+\\.[0-9]+|[0-9]+)', salary_string)\n",
    "            return np.mean([float(salary.replace(',', '')) for salary in matches ])*8*5*52\n",
    "    else:\n",
    "        return None\n",
    "\n",
    "# How paid (monthly, yearly, etc.) may affect outcome so I want to keep track of that \n",
    "def get_how_paid(salary_string):\n",
    "    if salary_string:\n",
    "        if re.findall('(.*) a year', salary_string):\n",
    "            return 'yearly'\n",
    "        elif re.findall('(.*) a month', salary_string):\n",
    "            return 'monthly'\n",
    "        elif re.findall('(.*) a week', salary_string):\n",
    "            return 'weekly'\n",
    "        elif re.findall('(.*) a day', salary_string):\n",
    "            return 'daily'\n",
    "        elif re.findall('(.*) an hour', salary_string):\n",
    "            return 'hourly'\n",
    "        else:\n",
    "            return None\n",
    "\n",
    "# Applying formulas\n",
    "ds_jobs_clean['how_paid'] = ds_jobs_clean.salary.apply(get_how_paid)\n",
    "ds_jobs_clean['annual_salary'] = ds_jobs_clean.salary.apply(get_standardized_salary)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# my cities have '+' in place of spaces because of how I had to input them into the URL, so I want to fix that\n",
    "ds_jobs_clean.search_city = ds_jobs_clean.search_city.str.replace('+' ,' ')\n",
    "\n",
    "# number of reviews has # and \"reviews\" so I want to clean that up and convert to numeric\n",
    "ds_jobs_clean.number_reviews = pd.to_numeric(ds_jobs_clean.number_reviews.str.replace(' reviews' ,'').str.replace(',' ,''))\n",
    "# To use number of reviews, I will have to fill NaN with 0 reviews, since ones with 0 reviews would just show up as Nans\n",
    "ds_jobs_clean.number_reviews.fillna(0, inplace=True)\n",
    "\n",
    "# sponsored has mutliple options, and I just want to have a dummy variable of whether something is sponsored or not\n",
    "ds_jobs_clean['is_sponsored'] = ds_jobs_clean.sponsored.apply(lambda x: 1 if x else 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#Converting time_since_posted to categorical\n",
    "def categorize_time_since_posted(time_since_posted_str):\n",
    "    categories = ['in the last day', '1-6 days ago', '7-12 days ago', '13-18 days ago', '19-24 days ago', '25-30 days ago', 'more than 30 days ago']\n",
    "    if re.findall('hour', time_since_posted_str) or re.findall('minutes', time_since_posted_str):\n",
    "        return categories[0]\n",
    "    elif re.findall('30\\+', time_since_posted_str):\n",
    "        return categories[-1]\n",
    "    elif re.findall('(.*) day', time_since_posted_str):\n",
    "        days_ago = int(re.findall('(.*) day', time_since_posted_str)[0])\n",
    "        if days_ago <= 6:\n",
    "            return categories[1]\n",
    "        elif days_ago <= 12:\n",
    "            return categories[2]\n",
    "        elif days_ago <= 18:\n",
    "            return categories[3]\n",
    "        elif days_ago <= 24:\n",
    "            return categories[4]\n",
    "        else:\n",
    "            return categories[5]\n",
    "#applying the function\n",
    "ds_jobs_clean['time_since_posted'] = ds_jobs_clean.date_posted.apply(categorize_time_since_posted)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# examine where location is not the same as the search city (since some have zip codes for example, must do a regex)\n",
    "# I want to use this information to determine whether a job is in the city or not (assuming jobs in the city would pay more)\n",
    "def in_city_proper(search_city, location):\n",
    "    if re.findall(search_city, location):\n",
    "        return 1\n",
    "    else:\n",
    "        return 0\n",
    "ds_jobs_clean['in_city'] = ds_jobs_clean.apply(lambda x: in_city_proper(x.search_city, x.location), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# convert \"start\" column to something that actually shows me what page something is on.\n",
    "def convert_to_page(start_value):\n",
    "    start = range(0,2000,10)\n",
    "    page = range(1,201)\n",
    "    d = {start:page for start, page in zip(start, page)} \n",
    "    return d[start_value]\n",
    "\n",
    "# applying\n",
    "ds_jobs_clean.ix[:,'page'] = ds_jobs_clean.start.apply(convert_to_page)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Categorical variable for expensive vs non-expensive cities\n",
    "expensive_cities = ['Washington, DC', 'San Francisco, CA', 'Boston, MA', 'New York, NY']\n",
    "ds_jobs_clean['expensive_city'] = ds_jobs_clean.search_city.apply(lambda x: 1 if x in expensive_cities else 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# creating a function that will recode into 'high', 'low', and 'no' ratings \n",
    "# choosing 4 as the cut-off for high vs low because about half the data is above, half below, and there's a big gap between the two groups\n",
    "def recode_star_rating(rating):\n",
    "    if np.isnan(rating) == False:\n",
    "        if rating >=4:\n",
    "            return 'high_rating'\n",
    "        else:\n",
    "            return 'low_rating'\n",
    "    else:\n",
    "        return 'no_rating'\n",
    "\n",
    "# applying the function\n",
    "ds_jobs_clean['rating_cat'] = ds_jobs_clean.star_rating.apply(recode_star_rating)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# I want to export the data for visualization in Tableau without the superfluous categories\n",
    "col = [x for x in ds_jobs_clean.columns if x not in ['location', 'salary', 'sponsored', 'start', 'date_posted']]\n",
    "\n",
    "ds_jobs_clean[col].to_csv('data_science_jobs2.csv', encoding='utf-8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "metadata": {
    "collapsed": false,
    "focus": false,
    "id": "634ab7c1-c76f-4b04-a36e-5ff3cb1f9eb2"
   },
   "outputs": [],
   "source": [
    "# load in the the data of scraped salaries\n",
    "jobs = pd.read_csv('data_science_jobs2.csv').drop('Unnamed: 0', axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Making a subset with only jobs with salary included so I can make a model.\n",
    "jobs_w_salary = jobs[jobs.annual_salary.notnull()].copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "median salary: 100000.0\n"
     ]
    }
   ],
   "source": [
    "# calculate median and create feature with 1 as high salary\n",
    "jobs_w_salary['high_paid'] = jobs_w_salary.annual_salary.apply(lambda x: 1 if x> jobs_w_salary.annual_salary.median() else 0)\n",
    "print 'median salary:', jobs_w_salary.annual_salary.median()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Visualization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div class='tableauPlaceholder' id='viz1479819998036' style='position: relative'>\n",
       "<noscript>\n",
       "<a href='#'>\n",
       "<img alt='Data Science Job Salaries ' src='https:&#47;&#47;public.tableau.com&#47;static&#47;images&#47;TB&#47;TB2QTWF6S&#47;1_rss.png' style='border: none' />\n",
       "</a></noscript>\n",
       "<object class='tableauViz'  style='display:none;'>\n",
       "<param name='host_url' value='https%3A%2F%2Fpublic.tableau.com%2F' /> \n",
       "<param name='path' value='shared&#47;TB2QTWF6S' />\n",
       "<param name='toolbar' value='yes' />\n",
       "<param name='static_image' value='https:&#47;&#47;public.tableau.com&#47;static&#47;images&#47;TB&#47;TB2QTWF6S&#47;1.png' />\n",
       "<param name='animate_transition' value='yes' />\n",
       "<param name='display_static_image' value='yes' />\n",
       "<param name='display_spinner' value='yes' />\n",
       "<param name='display_overlay' value='yes' />\n",
       "<param name='display_count' value='yes' />\n",
       "</object>\n",
       "</div>                \n",
       "<script type='text/javascript'>                    \n",
       "var divElement = document.getElementById('viz1479819998036');                    \n",
       "var vizElement = divElement.getElementsByTagName('object')[0];                    \n",
       "vizElement.style.width='1020px';vizElement.style.height='633px';                    \n",
       "var scriptElement = document.createElement('script');                    \n",
       "scriptElement.src = 'https://public.tableau.com/javascripts/api/viz_v1.js';                    \n",
       "vizElement.parentNode.insertBefore(scriptElement, vizElement);                \n",
       "</script>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "%%html\n",
    "<div class='tableauPlaceholder' id='viz1479819998036' style='position: relative'>\n",
    "<noscript>\n",
    "<a href='#'>\n",
    "<img alt='Data Science Job Salaries ' src='https:&#47;&#47;public.tableau.com&#47;static&#47;images&#47;TB&#47;TB2QTWF6S&#47;1_rss.png' style='border: none' />\n",
    "</a></noscript>\n",
    "<object class='tableauViz'  style='display:none;'>\n",
    "<param name='host_url' value='https%3A%2F%2Fpublic.tableau.com%2F' /> \n",
    "<param name='path' value='shared&#47;TB2QTWF6S' />\n",
    "<param name='toolbar' value='yes' />\n",
    "<param name='static_image' value='https:&#47;&#47;public.tableau.com&#47;static&#47;images&#47;TB&#47;TB2QTWF6S&#47;1.png' />\n",
    "<param name='animate_transition' value='yes' />\n",
    "<param name='display_static_image' value='yes' />\n",
    "<param name='display_spinner' value='yes' />\n",
    "<param name='display_overlay' value='yes' />\n",
    "<param name='display_count' value='yes' />\n",
    "</object>\n",
    "</div>                \n",
    "<script type='text/javascript'>                    \n",
    "var divElement = document.getElementById('viz1479819998036');                    \n",
    "var vizElement = divElement.getElementsByTagName('object')[0];                    \n",
    "vizElement.style.width='1020px';vizElement.style.height='633px';                    \n",
    "var scriptElement = document.createElement('script');                    \n",
    "scriptElement.src = 'https://public.tableau.com/javascripts/api/viz_v1.js';                    \n",
    "vizElement.parentNode.insertBefore(scriptElement, vizElement);                \n",
    "</script>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Word Feature Extraction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# creating a function that I can use to filter out many different types of jobs\n",
    "def word_filter(wordlist, title):\n",
    "    for word in wordlist:\n",
    "        if re.findall(word.lower(), title.lower()):\n",
    "            return 1\n",
    "    else:\n",
    "        return 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 187,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# categorizing job titles that suggest a higher-level job\n",
    "high_level_words = ['senior', 'lead', 'manager', 'director', 'president', 'sr', 'vp', 'vice', 'chief', 'executive']\n",
    "jobs_w_salary['is_high_level'] = jobs_w_salary.title.apply(lambda x: word_filter(high_level_words, x))\n",
    "\n",
    "# data with the word science may be important\n",
    "jobs_w_salary['ds_in_name'] = jobs_w_salary.title.apply(lambda x: word_filter(['data scien'], x))\n",
    "\n",
    "# Since machine learning is a pretty advanced part of data science, I thought it might influence salary.\n",
    "jobs_w_salary['ml_in_name'] = jobs_w_salary.title.apply(lambda x: word_filter(['machine learning'], x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 188,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Extract counts of top 20 features from Job Summaries\n",
    "summaryVectorizer = feature_extraction.text.CountVectorizer(max_features=20,stop_words=\"english\")\n",
    "summaryVectorizer.fit(jobs_w_salary.summary)\n",
    "\n",
    "summaryCol = [x + '_in_summary' for x in summaryVectorizer.get_feature_names()]\n",
    "\n",
    "summaryVectorDF  = pd.DataFrame(summaryVectorizer.transform(jobs_w_salary.summary).todense(),\n",
    "              columns=summaryCol)\n",
    "\n",
    "# Extract counts of top 20 features from Job Titles\n",
    "titleVectorizer = feature_extraction.text.CountVectorizer(max_features=20,stop_words=\"english\")\n",
    "titleVectorizer.fit(jobs_w_salary.title)\n",
    "\n",
    "titleCol = [x + '_in_title' for x in titleVectorizer.get_feature_names()]\n",
    "\n",
    "titleVectorDF  = pd.DataFrame(titleVectorizer.transform(jobs_w_salary.title).todense(),\n",
    "              columns=titleCol)\n",
    "\n",
    "# Consolidate\n",
    "df_with_wordcounts = jobs_w_salary.copy()\n",
    "df_with_wordcounts = pd.concat([df_with_wordcounts.reset_index(drop=True), summaryVectorDF, titleVectorDF], axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "focus": false,
    "id": "6e8a5a1c-1580-4845-a9b6-482bc00c73cd"
   },
   "source": [
    "## Predicting Salaries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 332,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def model_summary(model, X, y):\n",
    "    cv = model_selection.StratifiedKFold(shuffle=True, n_splits=5)\n",
    "    scores = model_selection.cross_val_score(model, X, y, cv=cv)\n",
    "    print 'mean cross-val accuracy score: ', scores.mean(), \"+/-\", scores.std()\n",
    "    \n",
    "    y_pred = model_selection.cross_val_predict(model, X, y, cv=cv)\n",
    "   \n",
    "    print metrics.classification_report(y, y_pred)\n",
    "    \n",
    "    y_score = model_selection.cross_val_predict(model, X, y, method='predict_proba')[:,1]\n",
    "    fpr, tpr, thresholds = metrics.roc_curve(y, y_score)\n",
    "    print 'AUC:', metrics.auc(fpr, tpr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 248,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X = pd.get_dummies(df_with_wordcounts.drop(['company', 'summary', 'title', 'time_since_posted', 'star_rating', 'high_paid', 'how_paid', 'annual_salary'], axis=1))\n",
    "y = df_with_wordcounts.high_paid"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Feature Selection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 308,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiYAAAGJCAYAAAC+bPjgAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAPYQAAD2EBqD+naQAAIABJREFUeJzs3Xd4XNW19/HvjLoty92We2e5N8A2YFoIoQUSWoCUGyCd\nS94QCAkhgQTSgEDqhZCbexOuE0IJnZiOKcbYYOOG23aXe5EtWb3NzPvHmZFkWbY1R6My0u/zPH48\ns8+Zo6WNsZd2WTsQiUQQERERaQ+CbR2AiIiISIwSExEREWk3lJiIiIhIu6HERERERNoNJSYiIiLS\nbigxERERkXZDiYmIiIi0G0pMREREpN1QYiIiIiLthhITERERaTeUmIiIiEi7kRrPzWYWAC4DPgOc\nAuQCYWAX8AHwHPCicy6U4DhFRESkEwg09RA/M7sa+AXQA3gNWAXsB1KAfsB04DSgEPipc+4fLRGw\niIiIdFxNSkzM7FmgO/AA8KpzruYo96UCnwW+AxQ45y5JYKwiIiLSwTV1KudvzrkXjndTNGF5CnjK\nzD7brMhERESk02nyVI6IiIhIS4tr8WuMmX0eeNc5t8PMfgxcDSwAvuOcq0hkgCIiItJ5xL1dOJqI\n/C8w1MxOA+4G3gfOAu5JaHQiIiLSqfipY3I98B/OufeBK4BFzrmvA18BrkxkcCIiItK5+ElMBgIL\no6/PBV6Nvt4O9ExEUCIiItI5+VljsgM4wcwygfF4NU0ATsdLTkRERER88ZOYPAz8C6gAVjrnFprZ\nDcD9wJ1+gjCzDOAhvKqyZcADzrnfHOXeS/EKvQ0BluEtuF1W73oh0A0IRJsiQDfnXJmf2ERERKT1\nxD2V45y7H7gWuA84J9pcCNwYvebH/XiVY88CbgB+YmaXNbzJzMYDj+IlJpOBFcDc6OgNZjYQLykZ\niVcuPxcYoKREREQkObR5HRMz6wLkA+c55+ZH234EnOOc+0SDe28CPu+cmxF9nw0UASc555aa2TnA\n/znnBrfqNyEiIiIJEfdUjpl1Bb6Ldy5OOnVTJgA0TCaaYEo0joX12t4Dbm/k3gPABDM7NXr/9cAh\nYFP0+nhgfZxfX0RERNoJP2tM/ox3Hs5rwJ4ExDAAyG9w/s5eINPMejvnDtRrfwK4BC9xCUV/XeSc\nOxS9Pg7oamZvAYa3BuUm59yGBMQpIiIiLcxPYnIxcLVz7t8JiqELUNmgLfY+o0F7b7x1IzcAHwDf\nAh4xs2nOuXxgLN6W5duA4ujvb5rZOOdcaYLiFRERkRbiJzEJA2sTGEMFRyYgsfcNF63ei7cT6GEA\nM/tGNJbrgF8D5wFpscWuZvYFvC3MFwOPNyWYSCQSCQQCx79RREREGmr2P6B+EpOn8Xbl3NHcLx61\nE+hjZkHnXDjalguUO+cKG9x7IvD72BvnXMTMVgDDou+rgep61yvNbAswqKnBBAIBiorKCYXCx7+5\ng0pJCZKTk9Xp+wHUFzHqhzrqC4/6waN+qBPri+byk5jsB75nZhcC62gwDeOcuz7O5y3HSyZm4Z25\nA16xtsWN3LsLb4FrfYY3rYOZbQTuds7Nib7vCoyJxtlkoVCYmprO/QcM1A/1qS886oc66guP+sGj\nfkgcP4nJLGBR9PXA5gbgnCs3sznAw2Z2PTAYuAX4MoCZ9QcORU8t/gvwNzNbgrcr52vAUGBO9HFz\ngbvMLA9vC/LPgG3AS82NU0RERFpe3ImJc+7sFojjZrzKr/Pwtv/e4Zx7PnptN97U0Rzn3JPRUZDb\n8aZnlgNnRxe+AtwKVOEVYesOvIm3a6dti7WIiIhIk/gqsBYtbPZFYBLeNMxq4AnnXFFiw2sTkYKC\n0k49JJeaGqRnz6509n4A9UWM+qGO+sKjfvCoH+pE+6LZi1/jLklvZkOBVcBvgFOBs/EWpK40M1Vc\nFREREd/iTkyAB/C24I5wzk1zzk0BRgB5eOfniIiIiPjiJzE5F7jZObc31hB9fSteHRERERERX/wk\nJjUcWfgMoJwjC6WJiIiINJmfxGQBcIeZpcUaoq9/FL0mIiIi4oufOiY/wKshsilaTwTgZKAbcGai\nAhMREZHOJ+4RE+fcOmAq8Bje1E0mXt2QKc65FYkNT0RERDoTPyMmOOfy8EZORERERBKmSYmJmc0D\nLnPOFZrZW8BRq7I55z6RqOBERESkc2nqiEkeEKr3WiXeRUREJOGalJg4566r9/ZOYIdz7rDau2aW\nCkxLYGwiIiLSyfjZLrwF6N1I+wjgneaFIyIiIp1ZU9eY3AB8L/o2ACwxs1CD23riTfOIiIiI+NLU\nNSaPAH3wRljuBJ4ESupdj0TfP53I4ERERKRzaeoakzLgbgAziwC/jraJtBuRSIR1eQX07pFFvx5Z\nbR2OiIj4EHcdE+fcXWaWamaDgJRocwCv2NrJzrlHExmgSFO9vmQHj7+5gUAATps4gEtOG04fJSgi\nIkkl7sTEzD4FzAH6NnK5HK8KrEirqq4J8dIib4lTJALvfbybhav3cObUgXz61OH0yNb5kiIiycDP\nrpxfAkuBi/BOGb4UuAkoAr6YuNBEmm7Bx3soKq0CYNLI3gQDAULhCPOW7uQHDy/kyXkbKS6rauMo\nRUTkePyUpJ8AXO+cW2lmy4FS59wfzawEb+fOcwmNUJLeW0t3sGLTgUavBQMBTrS+nDZpgO/nh8MR\nXvlgGwBD+mVz05WT2VdYzvPvbeGD1XuprgnzyofbeH3JdtLTUhp9Rv+eWVx82nCmju5DIBDwHcux\nfLh2L0+9vYnSippGr3fLSuPck4dw5tSBpKb4+ZlBRCT5+UlMQsCh6OuNwETgTWAe8ECC4pIOIm9P\nMX9/bf0x71m+MZ/CkkouOmW4r6+xxO1jX2E5ABfMGkogEKB/zy58/eIJXDhrGM/N38LS9fsJhSOU\nVzaeFGzdU8wfn/6Y0YO7c8WZozhhSA9fsRzN0vX7+fMLq4kco2ZyeWUNj76+ntcWb+OyM0Zx8rh+\nBFsoSRIRaa/8JCargEuAPwJrgdnA74HBCYxLOojYuo+01CBjBnc/4vqu/FIKS6p4+p3NpKWm8KmT\nh8T1/EgkwsuLvNGSPt0zOXlsv8OuD+6bzY2XTWLrniJWbT5IOHxkZlAdCvP+qj0UFFeyccch7nl0\nKVNH9+GyM0cyfEBOXPE0Zs3Wgzz8/CoiEeiamco5Jw4+IuGIAMs35JO3t5j9hRX8+YXVvPxBHlee\nNZoJI3o1OwYRkWThJzG5B3jKzKqAx4C7zGwuMBlv5EQEgL0Hy1ji9gFw1tRBXPPJMUfcc+BQBfc8\nupQDRRU8/uYG0lODnDVtUJO/xpq8AvL2FgNw3oyhpAQbnwIZnpvD8NyjJxkXnzqcN5fuYO77eZRV\n1rB8Yz4rNuYze/IApo/LpaysklDo8KSmS2Yqk0b2Ji316NMum3Yd4o9Pf0xNKEJGWgo3fW4KowYe\nmaABXHzacBav3cez725mX2E52/aW8MATyxk3rCczx/dvdPQkNTXAxBG9yc5KO2oMIiLJxM924efM\nbAYQcs5tN7PzgZuB5/GKr4kA8MqH24hEICUY4LwZjY+E9O6eya2fn8a9jy6loLiSOa860lKDTV5z\n8nJ0RCY7K43Zk/2vU0lPS+GCmcM4Y8pAXlqUxxtLdlBdE2b+yt3MX7n7qJ/r0z2TS88Y2WjisGN/\nCb97cgWV1SFSUwJ8+/JJR01KwFtvM3N8f060vryzfBcvLthCUVk1a/MKWJtXcNTPZWWkcuGsoXzy\npCFkHGUNjYhIsvC7wq6YaFLjnHsHb33J751zBxMVmCS3guJKFnzs/YN+yoRceuVkHvXefj2y+N7V\nU8npmg7AX19ay4dr9x73a2zdU8Sard4/2OeeNDgh/yh3zUzjyrNGc883TuGMKQNJCR57jUf+oQr+\n8uIa7vrbYlZuOkAkuohkf2E5DzyxnNKKGgIB+MYlExk/vGlTMqkpQc45cTD3fPMUPnv6CLIyjv3z\nQ3llDU+/s5nb/ryQt5fvJBQOH/N+EZH2LBA51mq8RpjZJ4EXgN84534cbVuItwj2AufcewmPsnVF\nCgpKqanpvH+5p6YG6dmzK83phyff2sgrH2wjAPzsqzMZ2KfrcT+zY38J9/1zGSXl1QQDAW64dCLT\nT2isXI7noedWsWTdPjLSUvj1Dae2yHRGBMjulklhYdlhfRGJwOqtB3nmnU3sLSivbR87tAfnzRjK\nP99Yz/7CCgCuv3Bcs0ZzQuEwVdWN/3fYsb+Ep9/ZzPrthbVt/Xt14fIzRnKi9U3YDqNE/JnoKNQX\nHvWDR/1QJ9oXzf5Lx28dk9qkBMA5dwreYth7mxuQJL/SimreWrYTgGkn9G1SUgLeQtVbrppKVkYq\n4UiEPz23ireW7mh0BGBvQRkfRdevnDl1YIutsUhLDdIlM42sjNTDfnXJTOXksf342Vdn8h/nGd2j\noz3rthXy+6dW1iYlV58zpllJCUBKMHjE14/9GjO4Bz/4/DRuunIyg/t6/bz3YBkPPbeKX/79IwqK\nK5vXASIircxPYjIB+N9G2v8HmNK8cKQjmLd0J5VV3uHTF84aFtdnh+V24+bPTSEjPYVQOMLfX1vP\nT/66mJWb8qk/uvfqB3XrV+LdyZNIqSneYt17vnEKl50xkqyMuumkS04b3iqxBQIBJo/qw0+vm8FX\nPz2O3tFps027injgieWUlFe3eAwiIoniJzHZD0xtpH0CUNhIu3QildUh3liyHYBxw3oycmD8221H\nDerOrVdPqx0B2JVfyu/+tZL7H1/Otr3FHCqp5L2P9wAwa0L/Y65faS0Z6Sl8+tTh3PvNU7n8zJFc\ne8FYPjN7RKvGEAwGOHXiAH759VlcMGso4PXdb59ccdT6LSIi7Y2f7cJzgD+ZWS/gg2jbycDPo9ek\nE3tv5W6Ky7yf0C88Jb7RkvpGDszhp9fNYMHHu3lm/mYOlVSxNq+Au/62mNzeXagJedM7F8z0/zVa\nQnZWmu9CcYmSlhrkijNHUVMT4fUl29myu4j/euZjbrpyMmmp/hYIF5dVEUlJQeXeRKSl+UlM7gb6\nAA8CaXgnC1fjrTHRduFOrCYUri0NPyy3G+OH9WzW84LBAKdPGciMcf159cNtvPzBNiqrQ+w+UAbA\n1NF9mrx+pbMJBAJcdc5oyiqrWfDxHtbmFfDw86u54dKJR6310pii0ipefH8rby/bSXpakDuvPZn+\nPbu0YOQi0tn5qWNSA9xgZt8HDC8p2eCcKz/2J6WjW7x2HweKvEWfF80alrAdIRnpKVwyewRnTB3I\nc/O3MH/lLlKCQS4+bXhCnt9RBQMBrr1gLOWVIZau38+yDfk88tI6rrto3HFL3ZdX1vDqh9t4dfH2\n2vVC5ZUh5i3dyTXnHFkoT0QkUZqUmJjZUGC7cy4SfR2zP/p7XzMDwDm3LbEhSjIIRyK15ef79+py\nzG2+fvXIzuDaC8ZyyWnDqQlH6NcjK+Ffo6NJCQb5xiXj+d2/VrI2r4AFq/aQlZnKNeeMaTRxrAmF\neXvZTl58f2vtlBxA9+x0DpVUsXDVHq48a5QOGRSRFtPUEZOtQC6wL/q6seIngWi7Sk92UOFIhHkf\n7WBfwZGDYyUV1ezMLwXggplDCR6nMFlztIfFrskkLTWFGy+bxP2PL2fL7iLeWLKDHftKGi3ctn1f\nCfmHKmrfTxjek8vPGkVZZYj7H/NqzKzYeIATLfGJp4gIND0xuRqIVXU9u4VikXZu5cYD/PONDce8\np0d2OqdMyG2liKSpsjJS+e7npnDvo0vZmV/Kum3H3kA3LLcbV5w1ignRarXBYIBeOZkcLKpgwce7\nlZiISItpamLyZ2ARsA34MvAd51xxi0Ul7dK26GF5gYB3RkxD6akpXHbmyGMeaidtJzsrjVuunspz\n8zc3OuoF3plBp07M5aSx/Q5bhxIMBjj7xME8/dZGPt58gKLSqtojBEREEqmpiUkK8Ckzex0vMfkv\nM8tv7EatMem49hZ4u2EG9enK3V+Z2cbRiB/eOp1xvj57zslDefqtjYTCERat2dumhe1EpONq6o+2\nfwf+G9gcfb8Y2NLg19bo79JBxc6E0XbRzmlI/261BfPe//joJy6LiDRHk0ZMnHP/aWYPA72At4DL\nqVtzIp3E3oPeiEm/XtoN01nNnjyAzbuK2LavhG17ixnav1tbhyQiHUyT65g45z4GMLPrgJecczod\nrBMpKa+mtMIra64Rk85r1oRc/vn6empCEd5ftUeJiYgkXFPrmNwJ3O+cKwOGAT+I1S1pyDl3d+LC\nk/Yitr4EoH9PjZh0VtlZaUwd3Yclbj+LVu/hCtU0kU7gYFEF763cXfvDWX3BIGRkplFZUU0jB6Ez\ncmAOM8f3b4UoO46mjphch1eCviz6+mgieCXr42JmGcBDwGXRr/GAc+43R7n3UuAXwBBgGd4OoWX1\nrl8D/AwYALwKfM05dyDemORw+w7W7eLo30sjJp3ZaZMGsMTtp6ismlWbDzJ1TJ+2DkmkRZRWVPPS\nwjze+GgH1TWNZB1NNGZwd9VfikNT15iMaOx1At0PTAfOAoYDc8xsq3Pumfo3mdl44FHga8D7wM3A\nXDMb6ZyrMLMZwP8AXwdW4J3f8whwcQvE3KnERkwy0lPorm2indrEkb3I6ZpOUWkVC1btVmIiHU5V\ndYg3P9rB3IV5lEVP5g4EoFe3TBo7zSGYEiQcajxxGTkwhx7dMloy3A7HzyF+mFkWEHLOVZnZOOAi\n4H3n3Ps+ntUF+ApwnnNuBbDCzO4DbgSeaXD7p4BVzrlHo5/9IfCfwHhgafT1E/WufwnIM7Nhzrk8\nP9+reGp35PTIStgZOJKcUoJBTpnQn1c/3M7yDfmUlFeTnZXW1mGJNFsoHGbBx3t4/r0tFBTXLaOc\nNqYPl50xkkF9s4/4TGpqkJ49u1JQUEpNM0ZVpE7ck8NmdgawE5htZgOAD4AfA++Y2ZU+YpiClyAt\nrNf2HtBYoYwDwAQzO9XMAsD1wCFgU/T6LODd2M3OuR14ReFm+YhL6qnbkaNpHIHTJg4AIBSO8MGa\nvW0cjUjzVVaH+OXfl/LIy+tqk5Ixg7vzwy9O59uXT240KZGW4WfV2i+B5/BqmXweKMI7R+f/Abf7\neN4AID96anHMXiDTzHo3uPcJ4CW8xKUKuA+4wjl3qN6zdjX4zF5gsI+4JCoSidSrYaKFrwKD+2Uz\ntL/3F/UC1TSRDuCFBVvYsrsIgEF9u/L/rpjMbV+YzpjBPdo4ss7Hz1TOdOA/nHPFZnYeMDe6vuMl\noNEFq8fRBWi49Tj2vuHEXG+8JOgGvJGabwGPmNk051z+MZ4V1wRfSiffZRD7/mO/F5VWUR6dZx3Y\npyupnajkfMO+6Kwa64czpgzkH6+tZ+ueYvYWlHWanyj1Z8LTkfphx74SXvtwOwCTR/Xm5qumNvkg\n0o7UD82VqD7wk5iUAulmlgnMBv4Sbc/Fm1aJVwVHJg6x92UN2u8FVjrnHgYws28Aa/F2Cv36GM9q\n+JxjysnRqADU9cOewrpcb/SwXvTs2bWtQmoz+jPhqd8P5582ksfe2EAoHOG597YyefSRi2C7Z2dw\n6uQBpKV2vEPH9WfCk+z9EA5H+NU/lhIKR0hPS+HbV02jd+/4/45L9n5oT/wkJm/hTaEUAGHgFTOb\nCvwhei1eO4E+ZhZ0zsVWDuUC5c65hkegngj8PvbGORcxsxV4tVViz2p4tG0uENdYc1FROaGjrLDu\nDFJSguTkZNX2w4a8ut3WXdOCFBSUtmF0rathX3RWR+uHqWP68JHbzwer9/DB6j2Nfnb8gp7cfPVU\n0jtIcqI/E56O0g9vLd3J2q1eIfPPzB5BRpC4/o7rKP2QCLG+aC4/icl/Ag8Dk4EvRKd0voQ3ZfJd\nH89bDlTjLVCN7eo5HW8NS0O78Hbg1Gd40zrgnYA8G5gDYGZD8NaXLIonoFAorNXV1PXD7gPe/6RZ\nGSlkpad0yr7RnwlPw364cNYw3LZCSsqrj/qZNVsLePDpj7nh0omkBDvOcLf+THiSuR+KSqt4ct4G\nwDuc9NyTBvv+XpK5H9qbuBMT59x+vLNy6rvNOXf0v5mO/bxyM5sDPGxm1+MlErfgnWKMmfUHDjnn\nKvCmjf5mZkvwdvF8DRhKNBEB/gS8ZWaLgCXA74AXtVW4efZGi6v169lFW4XlMCMG5PCH75ze6LVQ\nOMxDz65i2YZ8lm3I55GX1nHdReMI6s+QtBNPzNtQW831P843VTFuJ/zWMTkVWO+cy4+OllxlZguA\ne5xzER+PvBmv8us8vHUqdzjnno9e2w1cC8xxzj1pZl3xdv8MwhttOTu68BXn3KLoupOfAT3xKr9+\n3c/3KHVixdW0I0fikRIM8s3PTOB3/1rJ2rwCFqzaQ1ZmKtecMyahCe7qLQeZu3Ar+Ycq4v5sn+6Z\nfOHcEzrNwl2ps2brQRau9ra6nzFlgHbftCNxJybRf/gfAs41s3y8yqpv4k3jpAN3xftM51w53gLW\nI8rdO+eCDd7/DfjbMZ41h7oRFGmmw7cKq4aJxCctNYUbL5vE/Y8vZ8vuIt5YsoPszDQumd38AtJb\ndhfx1NubWJtX4PsZ+YcquP+J5fzwiyfSr4cS786iuibE3191AHTrksYVZ41u44ikPj8jJjcB33bO\nzTOzX+JVYv1UdOvww/hITKT9KiqtorIqBED/XvqLW+KXlZHKdz83hXseXcqu/FKee28LWZmpnHvS\nEF/P23uwjGfe3czidftq23K6pHHy2P5N3uIJUF5Vw3srd3OopIoHHl/GD794Ij2yVTq8M5i7MK/2\nB66rPzFGlYvbGT+JyQjgxejrc4GXo6/XcuSOGElysf95QSMm4l92Vhq3XDWVX/3jI/IPVfDYGxuo\nrgkzII5KwhFg1ZaDzF+xi1DYmzHOSE/hgplD+dTJQ8hMj/+vsyF9s3nszQ3sL6zggSeW84PPT283\n/0jtLyynJhRmgI+tq3J0uw+U8tIib9nhuGE9mTVBJ/+2N34Sk33AQDOrBqYBt0XbpwCN7xeUpBUr\nRQ86VViap2e3DL539VR+9Y+lHCqt4qm3Nx3/Q0eREgxw9vRBfPrU4eR08X+o5LknD6G0opoXFmxl\n5/5Sfv+vFdxy9VRfSU6i5B8q57n5W1i4yvvr9PqLxnHapAFtFk9HUl5Zw3+/sIaaUITUlABfOs+0\noL8d8vN/32N4J/yWAtuBt83sKryTfP83gbFJOxAbMemamdpufpKU5NWvZxduuWoqv/3XisMOSWuq\nADBzQn8uPX0kfRO0JuQzs0dQWlHDmx/tYNOuIh585mP+3xVTSGvlCsfFZVXMXZjHvKU7qAnV7SH4\n20vr6JKRyrQT+rZqPB1NVXWIPzy1kry9xQBcctoIcvXDVrvkJzH5IbADGAk86JwLmVk/tL6kQ4rt\nyOmnaRxJkMH9srn3m6dQ6CMxycxIfIIcCAS45pNjKKuoYeHqPazeWsB/v7Cab352QqvUXamsCvHa\n4m288uE2yiu99VzBQIBTJvZn+YZ8Sitq+NPzq/nu56YwbljPFo+nI6oJhfnTc6tw272anadPHsBF\npww7zqekrfipYxLGGx2p3/bHo9wuSS5Ww0QLXyWRUlOC9GlHu2CCgQDXXTiW8soalm/M56P1+7n7\nkSV0zTzyr8hAIEBaWgrV1SEiET/VEQ6360AZRaVVte9Psr5cesZIBvTuyqadh7j/8eVUVof4w9Mr\nufXqaYwcmNPsr9nebNtbzCsfbqNH1wzOnzmUnK7+p+caCkci/PWltazY5FWwPsn68uXzx2oKpx3z\nW8fkEmASEKsxHcA7k+Zk59y5CYpN2lgkEmFfYayGiUZMpGNLTQnyrc9O4LdPrmDdtkK27ytp1a8/\ndmgPrjhr9GGJx6hB3bnx8kn8/l8rqKwK8dsnl3PbF09kUJ+OsSB2X2E5z727mUVr9ta2vbV8J+fP\n8BY0Z2U0b61PJBLhn6+vZ1G0XsmEEb342sUT4tq9Ja3PTx2Te4DvA3uBfnjn0/SPPuuxhEYnbaqg\nuJKqaq/Ecj8VV5NOIC01hW9fPpm5C/Nqj2JoKNEjJulpKZw6MZeJI3o1+lP8hOG9+MYlE3jouVWU\nVtTUbm1O1BqbtlBUWsWL72/l7WU7a3dYpaUGqa4JU1kV4vn3tjBv6Q4uOW0EZ04d6Lsi67PztzBv\n6U4ARg3K4cZLJ7X62iGJn5909AvATc65P5jZdryzaUqA54DNiQxO2tZhO3I0YiKdRFZGKlecNeqo\n11NTg/Ts2ZWCgtJWOxvlROvHtReM5W8vraOwpIoHHl/ObV+cHlfdldIK79SQrpmJWaNTXFZFYXlN\n9PC6piVokUiEZRvyeeXDbbX1kVKCAc6YOpBLTh1OQUklT729iTVbCyguq+bR19fz2uJtfHb2SAb3\ni68674qN+fz7/a0ADO6bzU1XTiEjvWMcJNnR+UlM+gMvRF+vBGY4554ys9uBvwJ3Jio4aVt7Dtar\nYaI1JiJt6vTJAymvqOHxeRvZV1jO3Y8s5jOzRzB78oBjLtItKa/mpYV5vPHRDgDOmjaQi04ZTnef\n6zgOlVbx4oItvLO8rp6MXzPG9ePSM0bW/uDTPTuD7109jdVbD/LU25vI21PM/sIK/vLvNb6/Rr8e\nWdxy1ZSEJWTS8vwkJgVALHXdCEwAngK24Z1fIx1EbEdOdlaa/qcWaQc+NWMoZZU1vLBgK4UlVfzf\nK45XP9zO5WeOZPoJfQ+bCqqsDvHGku28tGgb5ZU1te1vLNnBuyt2ce5JQzh/5tAm/79dXlnDKx9s\n47XF26msDjXr+5gwohdXnDmKYbndGr8+vBfjvtyTJev28cw7m9lXWN7ofcfTKyeDW66eSndV9E0q\nfhKTt4B7zezrwAfA7Wb2IHAFsD+RwUnbik3l6PA+kfbjs6ePZMSAHJ56ZxM795ey52AZDz67ipED\nc7jyrFGMHtyd91bu5vn3tlBYUrfbZ9qYPqSmBFm8bh9V1eFozZSdnDdjCOeedPSFptU1Yd5etpMX\n399KSXndIfKnTMjlwtkjKCutbPJUDnhn0zTl0MRgIMCMcf2ZfkJf8vYUUx3ntFkgACMH5pCWqumb\nZOMnMbkIfRLJAAAgAElEQVQVbyrnc8CDwC14C2HBOyVYOog9B1XDRKQ9mjK6D5NG9mbh6j08N38z\nB4oq2byriHv/uYxuXdIoLqtLIE4Y3J0rzhrN6MHdAbhobzHPvruZFZsOUF5Zw3Pzt/DGkh0My+1G\nSjBASjBAMPp7SjDAhh2HDju5eeKIXlx+5ihGDe7eKmttUlOCjBrUvcWeL+2Pnzom24FpZpbpnKsy\ns9OB84HtzrnFCY9Q2kQ4HGFfgWqYiLRXwWCA0yYNYMa4fry1dCf/XphHSXl1bVIyuG9XLj9zFJNH\n9T5simdo/25858opbNp5iGfe3czavAJKyqtZveXgMb/eiAHduOKs0SryJi3O9yZx51xF9Pcy4JmE\nRSTtQv6h8tqhU+3IEWm/0lJT+NSMocyePJBXP9zGhh2FnDZpAKdMyD1mvY5Rg7pz6zXTWJtXwLsr\ndlFWUUM4HCYUjhz2Kys9hU9MH8yJ1ldFyaRVNCkxMbMw3uGex+Wc04ReB7B7f10NB42YiLR/XTJT\nufSMkXF/btywnhoFkXalqSMm19PExEQ6hl35dVUvNWIiIiKtpUmJiXPukfrvzaw70Mc5tyn6/jLg\nbefcsScpJWnsyvdGTHK6pDW7LLSIiEhTxV2b18ymA5uAb9VrfgBYZWYTEhWYtK1d0amcfjoWXERE\nWpGfQwN+g7dd+Ef12sYArwK/TURQ0vZiUzmqYSIiIq3JT2JyIvBz51xlrME5VwP8CpiZqMCk7YTD\nEfYc0KnCIiLS+vwkJsVAY0u/BwKVjbRLkjlwqIKaUHSrsKZyRESkFflZ1fg08JCZfQuvJD3AyXhV\nYFXPpAPYc9ipwprKERGR1uMnMbkNGAW8zuFbiJ/FK1cvSS52eB9APyUmIiLSivyUpC8FLjSzE4BJ\nQDWw1jm3IdHBSduIjZj0yE4nM11bhUVEpPU0pyT9emB9AmORdqL2VGGtLxERkVbmZ/GrdHB7DnqH\n9+UqMRERkVamxEQOU15ZQ35h7FRhJSYiItK6lJjIYVZtOUgo7K1p1sFeIiLS2vyUpL/TzI74UdrM\ncszsd4kJS9rKsg37AeiVk8GIgTltHI2IiHQ2TVr8amZjgX7Rtz8BVphZQYPbJgFfB25KXHjSmmpC\nYVZuPADAjAkDCAYChHWotIiItKKm7soZBbxIXd2SZ49y31+bHZG0mfXbCymrrAFg1sTcNo5GREQ6\noyYlJs65uWY2HG/qZzMwA9hf75YIUOKcO5jwCKXVLFufD0BmegqTR/ehpLiijSMSEZHOpslrTJxz\n25xzW4ERwCYg1TmX55zLA05qofiklUQiEZZt9HLNyaP7kJaa0sYRiYhIZ+RnV05vYAPwrXptDwCr\nzGxiQqKSVrdtbwkHi7wzGKef0LeNoxERkc7KT2LyG+AF4Ef12sYAr0avSRKK7cZJCQaYMrp3G0cj\nIiKdlZ/E5ETg5865yliDc64G+BUwM1GBSetaGl1fYkN70DUzrY2jERGRzspPYlIMjGykfSBQ2Ui7\ntHP7C8vZsb8EgGljNI0jIiJtx88hfk8DD5nZt4APom0nAw8CzyQqMGk9yzbk176eNqZPG0YiIiKd\nnZ/E5Da8uiavw2HVt54Fbk1EUNK6lkfXlwzL7UavnMw2jkZERDqzuBMT51wpcKGZnYBX7bUaWOuc\n2+A3CDPLAB4CLgPKgAecc0cspDWzt4AzG3nEX51zX43eUwh0AwLRaxGgm3OuzG98HVlJeTVueyGg\n0RIREWl7fkZMYiqAEuBdvESgOe4HpgNnAcOBOWa21TnXcGroUiC93vtZwBN400iY2cBoLCOB8thN\nSkqObsXGfCLRca/pWl8iIiJtLO7ExMzSgTnA54AwcAJwv5l1Ay53zhXF+bwuwFeA85xzK/DO4bkP\nuJEGa1acc4X1PhcEfgnc65xbFm0eB+yOFn2TJoitL+nTPZNBfbu2cTQiItLZ+dmV82NgCvAJvFET\ngD8Ao4F7fDxvCl6CtLBe23scf+vxdUBP4L56beOB9T5i6JSqqkOs2uId2jf9hL4EAoHjfEJERKRl\n+UlMrgG+7Zx7m+ji1+jrrwKf8fG8AUB+tBZKzF4g08yOVenr+8BvG0zTjAO6mtlbZrbLzOaa2Rgf\nMXUKa7YWUFUdBrS+RERE2gc/a0wGARsbad8G9PLxvC4cWf8k9j6jsQ+Y2dnROP6nwaWxeKMot+HV\nW7kNeNPMxkUX7TZJSoqffC35LN/oTeNkZ6UxdnhPUoLe9x37/jtLPxyL+sKjfqijvvCoHzzqhzqJ\n6gM/icka4JMcmRRcHb0WrwqOTEBi74+2aPVy4OX6a06izgPSYqMoZvYFYDtwMfB4UwPKyclq6q1J\nKxSOsGKTl5jMnJhLn95Hrl/uDP3QVOoLj/qhjvrCo37wqB8Sx09i8lPgCTMbH/38l83MgCuAq3w8\nbyfQx8yCzrlwtC0XKG8k8Yg5H/hJw0bnXDXe9uXY+0oz24I3utJkRUXlhELh49+YBIpKq9hbcGR+\ntzu/jEMlVQBMHN6TgoK6AaWUlCA5OVkdqh/8Ul941A911Bce9YNH/VAn1hfN5aeOyb/N7HLgdiCE\nV1RtFXCVc+5pHzEsx0smZgHvR9tOBxY3dnN03clIYEEj1zYCdzvn5kTfd8U7YHBdPAGFQmFqapL/\nD1h+YTm3/+UDao7xP0t6apCxQ3s2+v12lH5IBPWFR/1QR33hUT941A+J42e78PeAx5xzZyQiAOdc\nuZnNAR42s+uBwcAtwJejX68/cMg5F9sBNBFvNGVrI4+bC9xlZnlAPvAzvLUvLyUi1mSzcvOBYyYl\nADPH9ycjLaWVIhIRETk2P1M5PwaeS3AcN+NVfp0HHALucM49H722G7gWr3YKQH/gaFM8twJVwKNA\nd+BN4CLnXOQo93doW3Z7JWV652Ryw6UTj7iemhJU7RIREWlX/CQmHwCXAEeUjPfLOVeOV5fkukau\nBRu8fxJ48ijPqcJLTnRmD7B1dzEAowblMGJAThtHIyIicnx+EpNDwK/N7HZgA/VKvwM45z6RiMCk\neSqqath1wFvQOjxXSYmIiCQHP4lJKXXTKtJO5e0prj0DZ8SA5h5lJCIi0jr8JCavAa845woSHYwk\nzpboNE4gAMNylZiIiEhy8FOm7UG8MvLSjm3d4y18Hdi7K5npzTlEWkREpPX4SUzWA5MSHYgkVmxH\nznBN44iISBLx86P0CuBRM7uVxhe/Xp+IwMS/kvJq9hd6ZV+0G0dERJKJn8TkBGB+9HVuAmORBNka\nHS0BJSYiIpJc/JSkP7slApHEiU3jpAQDDO6b3cbRiIiINJ2vVZFmlg18EW+tSTWwGnjCOVd0zA9K\nq4jtyBnSL5u0VB3FLSIiySPuf7XMbCjeoX2/AU4FzgZ+D6w0s8GJDU/iFYlEakdMNI0jIiLJxs+P\n0w8A24ERzrlpzrkpwAggD7gvkcFJ/AqKKzlUWgVoR46IiCQfP4nJucDNzrm9sYbo61uB8xIVmPgT\nm8YBjZiIiEjy8ZOY1ABljbSXAxnNC0eaK1ZYLSMthYG9dXKwiIgkFz+JyQLgDjNLizVEX/8oek3a\nUGx9ybD+2QSDgTaORkREJD5+duX8AFgIbDKzJdG2k4FuwJmJCkziF4lE2BqdyhmuaRwREUlCcY+Y\nOOfWAVOAf+JN3WQCjwJTnHMrEhuexGNfQTlllTUAjByoxERERJKP3yIXGcC/nHMXOecuBHYDacf5\njLSwLfUqvmrEREREkpGfOiafxDsv59J6zVcDy8xsdqICk/jFduRkZ6XRt3tmG0cjIiISPz8jJr8E\nfuOc+3GswTl3CvBH4N5EBSbx2xLdkTM8txuBgBa+iohI8vGTmEwA/reR9v/BW3sibSAUDrNtjxa+\niohIcvOTmOwHpjbSPgEobF444teu/DKqasIAjFDFVxERSVJ+tgvPAf5kZr2AD6JtJwM/j16TNlB/\n4asqvoqISLLyk5jcDfQBHsTbiRPAO2H4j8AdiQtN4rE1mpj07JZBj2wV4BURkeQUd2LinKsBbjCz\n7wOGl5RscM6VJzo4abrYjpzhuZrGERGR5OVnxAQA51wJ8FECYxGfqmtC7NhfAmgaR0REkpvfAmvS\njmzbV0IoHAGUmIiISHJTYtIBxM7HARiuHTkiIpLEmpSYmNl0M0tp6WDEn9iOnH49s+iaqZMBREQk\neTV1xORtYACAmc0zsx4tFpHELZaYaBpHRESSXVMXv1YDXzWzt4CzgDPNrKCxG51z7yYoNmmCQyWV\n7D5QBsAonSgsIiJJrqmJyW/w6pfcCUSAZ49yXwTQlE8rWrO1Lj+cMKJXG0YiIiLSfE2aynHO/QLo\nCYzAK6g2I/q64a+RLROmHM2qLQcBr7Babq8ubRyNiIhI8zS5jolzrggoMrOzgeXRQmvShiKRCGu2\neonJhBG9dKKwiIgkPT+VX98xs0vM7HZgEt76kzXAr51zR5vikRawc38ph0qrAJgwXNM4IiKS/OKu\nY2Jml+GtMdkF3A7cBewF/mVmlyQ2PDmW1dHRkgAwfnjPtg1GREQkAfyUpL8DuNs5d1e9tt+a2Z3A\nj4AXEhKZHNfq6PqSof270a1LehtHIyIi0nx+Kr+OBR5tpP0xvKkdaQXVNSHWby8EtBtHREQ6Dj+J\nyS5gdCPtY4DC5oUjTbVhxyGqasIATNA0joiIdBB+pnL+CTxsZt8CFkTbZgMPAU8kKrDOqCYU5sUF\nWwkGA1x82nCCx9hlE1tfkp4aZPRgFeIVEZGOwU9i8gu8KZu5eAXVwFt/+W+8xbBxM7MMvMTmMqAM\neMA595tG7nsLOLORR/zVOffV6D3XAD/DK6H/KvA159wBP3G1pnAkwl/nrmXRmr0A5Pbqwszx/Y96\nf2x9yQlDe5CWqrMYRUSkY/CzXbgC+KyZjcVLUALASufcumbEcT8wHa/c/XBgjpltdc490+C+S4H6\nqzxn4Y3SPAhgZjOA/wG+DqwA/gg8AlzcjNhaXCQS4Z+vr69NSgDmLsxjxrh+jdYmKSqrYtveEgAm\napuwiIh0IH5GTACIJiLNSUYAMLMuwFeA85xzK4AVZnYfcCNwWGLinCus97kg8EvgXufcsmjzfwJP\nOOcejd7zJSDPzIY55/KaG2tLeXb+FuYt3QlAdlYaJeXV7NhfwspNB5gyus8R98eKqgGM18JXERHp\nQNrDHMAUvARpYb2294CZx/ncdXhl8u+r1zYLqD1E0Dm3A9gWbW+XXv1wG/9+fysAg/t25WdfmUH3\nbG9QaO7CPCKRyBGfWbPFOx+ne3Y6g/p0bbVYRUREWlp7SEwGAPkNStzvBTLNrPcxPvd94LfOubIG\nz9rV4L69wOCERJpg81fs4ol5GwHo1yOLW66aSvfsDM47eSgAG3ceqt0SHBOJRGoXvk4YrjL0IiLS\nsfieykmgLkBlg7bY+4zGPhA9r2cQ3nqSpjyr0eccTUpKy+dri9fu5ZFXvJmwnt0y+MEXp9O7RxYA\n55w0mLkLt1JaUcPLH2xjwsi6/GxnfikFxd63OHlUb1JbYOFr7PtvjX5o79QXHvVDHfWFR/3gUT/U\nSVQf+E5MzCwTOAHvH/7Nzrlqn4+q4MjEIfa+jMZdDrxcf83JcZ51tOc0KicnK57b47Zi/X7+9Nxq\nIhHo1iWNn3/zVIbm5tRe7wlcfPooHn/dsXLTAQ6UVtduCZ7/8Z7a+06dNpie3TJbLM6W7odkor7w\nqB/qqC886geP+iFx4k5MzCwduBf4JnU7ZMrN7F7g5865IxdFHNtOoI+ZBZ1z4WhbLlDeSOIRcz7w\nk6M8K7dBWy6wO56AiorKCYXCx7/Rh+qaMA/88yNqQmEy01O45eqpdMtIoaCg9LD7Tp/Un2fe3kBV\ndZjHXlnLjZdPBuDD1V5iMrR/NtSEjvhcIqSkBMnJyWrRfkgW6guP+qGO+sKjfvCoH+rE+qK5/IyY\nPABcAXwb+ABIAU4Hfhp9XmMJw7EsxzuheBbwfrTtdGBxYzdH152MpK64W32L8Iq9zYneOwRvfcmi\neAIKhcLU1LTMH7D5K3bVTsVcf+E4hvbr1ujXykpP5aypg3ht8XYWr93Hjn0l9Omeybo8b+Hr+GG9\nWizGmJbsh2SjvvCoH+qoLzzqB4/6IXH8JCZXA192zr1Ur225me0C/os4ExPnXLmZzcGrJns9XiJx\nC/BlADPrDxyK1k8BmIg3mrK1kcf9CXjLzBYBS4DfAS+2l63C4XCElxd5oQzq05Xp1veY93/q5CG8\n+dEOQuEILy3K47SJuVRWhwCdjyMiIh2Tn5Uq6cCWRtrXAtk+47gZ+AiYh1cU7Q7n3PPRa7uBz9W7\ntz9HOZPHObcI+AZecvQecAC43mdMCbd0/X72FpQDcOGsYccsOQ/QKyeTUyd6M1MLV+1h/kpvRio1\nJciYwd1bNlgREZE24GfE5P+AO83sy865qnrt38M7YThuzrlyvLok1zVyLdjg/ZPAk8d41hyiUznt\nSSQSYW50tKR3TiYnj+vXpM9dMGsY763cTSgc4f1V3voSG9Kd9LSUFotVRESkrTQpMTGzeQ0+Mxs4\n3cyWACFgGjAEeL6RjwuwJq+AvD3FAJw/cyipTdxWldurCyeN7cfidftq21TtVUREOqqmjpg0XKOx\nqcH7dxIQS4f20kKvC7t1SWP25AFxffbCWcMOS0wm6HwcERHpoJqUmDjnjphikabbvKuItdHdNJ88\naQgZcU7DDMvtxsSRvVi1+SA5XdMZ3M/vUh4REZH2zVeBNTMbhre9Nx3vdOGYiHPu74kIrCOJ7cTJ\nSE/hE9MH+XrGteeP5Zl3NzNzfP/jLpoVERFJVn4KrH0Nb1tuY4skIoASk3p2Hyhl6fr9AJw9bRBd\nM9N8PadXTiZf/fT4RIYmIiLS7vgZMbkdLzH5kXOuKMHxdDgvL9pGBEhNCXDuSUPaOhwREZF2zU8d\nkwHAA0pKju9gUQULoyXkT504gJ7d4jpLUEREpNPxk5gsByYkOpCO6LXF2wmFIwQCcMGsoW0djoiI\nSLvnZyrnPuBBMxsJrMM7XbiWc+7dRASW7ErKq3ln+S4ATrJ+9O/ZpY0jEhERaf/8JCZPRX//fSPX\nIniH+nV681fsqj3X5sJZw9o4GhERkeTgJzEZkfAoOqCPojtxRg/qzrDcbm0cjYiISHJoakn6gc65\nXQBNPanXzAY553Y2J7hkVVBcyeZd3trg6Scc+wRhERERqdPUxa9vmNkdZnbcI23NrJ+Z/Rx4s3mh\nJa/lG/bXvp5+Qp82jERERCS5NHUqZybwa2BX9EC/ucDHwD68NSV9genAJ4Bz8U4ZnpXwaJNErKDa\n4L5d6adFryIiIk3W1LNyioFvmtn9wE3Aj4BBeItdwStLvx3vdOGpzrn1LRBrUiirqGbdtkIApo3R\nNI6IiEg84lr86pzbCNwI3Ghmg4FcIAzsds7tboH4ks6KTQcIhb18TetLRERE4uPrED8A59wOYEcC\nY+kQlkWncXrnZDC0v04BFhERiYefyq9yFFXVIT7efBDwpnECOgVYREQkLkpMEmhNXkFtUTVN44iI\niMRPiUkCxXbjZGelMWbIcXdWi4iISANKTBIkHI6wfEM+AFNG9yYlqK4VERGJl6/Fr2Z2AfB9wIBT\ngOuAjc65fyQwtqSycechSsqrAZiubcIiIiK+xP1jvZmdCzwL5AE98QqspQGPmNl/JDa85BGbxklP\nCzJhRK82jkZERCQ5+ZlvuAu4zTl3LVAD4Jz7EXA7cGviQksekUikNjGZOKI36Wk6YFlERMQPP4nJ\nJODFRtr/BYxqXjjJafu+EvIPVQAwbYzOxhEREfHLT2JyCBjYSPsE4GDzwklOy6KLXoOBAFNGKzER\nERHxy09i8ijwOzObjHdWTraZnQ/8F/BEIoNLFrFpHBvag+ystDaORkREJHn52ZXzY2AIsDz6fhne\nIX7/xjvcr1PZX1jO9n0lgIqqiYiINJefxGSoc+7zZnYHMA1v1GWVc25NYkNLDrGzcUDrS0RERJrL\nT2Lyrpld6pz7ENiU6ICSzdLo+pLhud3olZPZxtGIiIgkNz9rTKqjvzq90opqNuwoBGCapnFERESa\nzc+IySPAK2Y2B9gIlNe/6Jybk4C4kkJ+YQWRiPd69CCdjSMiItJcfhKTO6O/39LItQjQaRKT4vKq\n2tc5XbQbR0REpLniTkycczqdLqq4rG5Gq1uX9DaMREREpGPwdYgfgJmNxasCWwWsdc6tT1hUSSKW\nmARA9UtEREQSIO7ExMwygX8Cn63XHDGzF4GrnHOViQquvSsu86ZyumalEQwG2jgaERGR5OdnWuYX\nwAy8xKQn0Bu4DJgO/DRhkSWB2IhJN60vERERSQg/UznXAF93zv27XtvzZhYCHgJ+mJDIkkBJeTQx\n0TSOiIhIQvgZMekGrGuk3QGdqphHbCpHC19FREQSw09isgq4spH2z+ElJ51GbConW1M5IiIiCeFn\nKufneFM3U4EF0bbZeOtMrvEThJll4E0DXQaUAQ84535zlHsnRe89EdgAfMc593a964V4ozqx1agR\noJtzrsxPbMdSN2KixERERCQR4h4xcc7NxRsxGQb8CrgHGAp8zjn3L59x3I+3ePYs4AbgJ2Z2WcOb\nzCwHeA1v1GYi8CzwrJn1iV4fiJeUjARyo78GtERSEgqHKa2oAaBblqZyREREEsFvHZPngPnOuXwA\nM5sBfOTnQWbWBfgKcJ5zbgWwwszuA24Enmlw+7VAsXPuW9H3PzWzC4CTgFeAccBu51yen1jiUVJe\nU/taIyYiIiKJEfeIiZmNwltL8v16zXPxEoohPmKYgpcgLazX9h4ws5F7zwSer9/gnJvpnHsl+nY8\n0CqF3mLTOKDFryIiIoniZ/Hr7/DWdvy2Xtt4YBvQ6LqQ4xgA5Dvnauq17QUyzax3g3tHAvlm9mcz\n221m75vZqfWujwO6mtlbZrbLzOaa2RgfMR3X4eXoNWIiIiKSCH6mck4HZjrndscanHP7zexWYL6P\n53UBGlaLjb3PaNCeDfwA+D1wPt5i29fMzJxzO4GxeEXfbgOKo7+/aWbjnHOlTQ0oJeX4+Vp5ZV0e\n1aNbBqmpHecIodj335R+6OjUFx71Qx31hUf94FE/1ElUH/hJTKrx/vFvqAt1O2HiUcGRCUjsfcNF\nqzXAMufcXdH3K8zsU8CX8BbhngekxRa7mtkXgO3AxcDjTQ0oJyfruPfU1PtWhw7qQVpqSlMfnzSa\n0g+dhfrCo36oo77wqB886ofE8ZOYvAz8wcyucc5tAjCzkXhTO68c85ON2wn0MbOgcy4cbcsFyp1z\nhQ3u3c2Rxd3WA0MAnHPVeIkT0feVZrYFGBRPQEVF5YRC4WPesze/BIDM9BRKiivieXy7l5ISJCcn\nq0n90NGpLzzqhzrqC4/6waN+qBPri+byk5h8D3gdWG9mBdG2nni7cr7r43nL8ZKJWcD70bbTgcWN\n3LsIOKNB21jgHwBmthG42zk3J/q+KzCGxivVHlUoFKam5th/wA6VeLNN3bqkHffeZNWUfugs1Bce\n9UMd9YVH/eBRPyRO3ImJc26fmU0HPolXS6QaWAO86ZyL+HheuZnNAR42s+uBwcAtwJcBzKw/cMg5\nVwE8DNxoZncCj0bvGRF9Dd7uoLvMLA/IB36Gtyj3pXjjOp66A/y0I0dERCRRfNUxcc6FgFeBV80s\nDW/Lb1egxGccN+NVc50HHALucM7FtgXvxqtfMsc5t83MzgP+iLewdS1wYb2FuLcCVXiJSnfgTeAi\nPwnT8dRWfdUBfiIiIgkTd2ISrVXyv8CPgY/xplzGAwfN7JPOueXxPtM5Vw5cF/3V8FqwwfuFeAXV\nGntOFV5ycmu8McSruFwjJiIiIonmZ2/Pb/FGI/bhHdw3DO+snGeB+xIXWvtWN5WjERMREZFE8ZOY\nfAL4hnNuK3Ah8LJz7n3g18ApCYyt3YpEIpRojYmIiEjC+UlM0vCmbQLAOcAb9Z5Vc9RPdSBllTWE\nI96yFY2YiIiIJI6fxa/L8A7d2423TfglM0vHW4wa9/qSZFS/HH22Fr+KiIgkjJ/E5Bbg30Af4F7n\n3A4zewj4DF6Z+A5PB/iJiIi0jLincpxzi/EO3uvtnLs92vw7YJRz7oNEBtde6QA/ERGRluG3jkkY\nKKz3fn3CIkoCh4+YKDERERFJFB2H6ENsxCQtNUhGWsc7vE9ERKStKDHxoX4Nk0DAz4HKIiIi0hgl\nJj6UlMfK0Wvhq4iISCIpMfFBVV9FRERahp+zcnKBnwOnAenAYXMZzrmRiQmt/YolJtlKTERERBLK\nz66cvwAnAo/jnQTc6RRrKkdERKRF+ElMPgGc75ybn+hgkkEkEtFUjoiISAvxs8akBNib6ECSRWV1\niOqaMKDEREREJNH8JCZzgO+bWacs4HF41VdN5YiIiCSSn6mcPsA1wKfNbBNQWf+ic+4TiQisvVI5\nehERkZbjqyQ98FhCo0giOsBPRESk5cSdmDjnrmuJQJJFSblGTERERFqKrxETMxsC/CcwCagGVgN/\nds5tS2Bs7VJsKicYCJCV4XfASURERBoT9+JXM5sErAS+BFThFVi7FlhpZhMSGl07FJvKye6SRlDn\n5IiIiCSUnx/5fw28BXzeOVcBYGaZwKPAvcCnExde+6MaJiIiIi3Hz3bh2cBPYkkJQPT1XdFrHVps\nxKRblhITERGRRPOTmBTjnZHTUKfYolJcHhsx6RTfroiISKvyk5i8CfzazHrFGsysD3Bf9FqHVjti\noqkcERGRhPOzxuQ24H1gm5mtj7adABwAzkxUYO1V3RoTjZiIiIgkWtwjJs65HcB44FZgIbAI+B4w\nyTm3NaHRtTPVNWEqqkIAZGuNiYiISML5KsTh3P9v787D5KrKPI5/O8FsQJwISsKgLII/gmIExDAY\nVkdBGPZBREbRKDosI5uKILIzbGGRHUGWDMo2siqDiqDsggtBFF5EiEDASIBAAklISM8f51T6pqh0\nqk5d05UAABvJSURBVG9XJ5W+v8/z5EnVubdPvfd0ddfb55x7TswELmhxLG3Pi6uZmZn1raYSE0lP\nARtHxEuSngY6F3VuRKzVquDajZejNzMz61vN9phcAczKjy/vm1DanzfwMzMz61tNJSYRcWzh6Z3A\n/RExt3hOXmRt+xbG1nbcY2JmZta3ytwufCfwTw3K1wOu7F047a3YY7LCUO+TY2Zm1mrNzjE5CDg9\nP+0A/i6p0akPtiiutjRjVuoxWX7IcgwcUCanMzMzs+40+2f/ucDLpB6WS4GDgVcLxzuBmcAdLY2u\nzXgNEzMzs77V7ByTecBEAEmdwNURMacvA2tHM3NisoInvpqZmfWJHk+UiIgrJL1b0geAgbm4AxhM\nuqX4xFYG2E68gZ+ZmVnf6nFiImkv4BK6Nu3roGtdk8lA/01MvIGfmZlZnyozg/M7wNXAB0nzTDYG\ndgaeB45uXWjtp2uOiXtMzMzM+kKZxGQt4JSIeByYBLw7Im4BDgQOamVw7WT+/E5ed4+JmZlZnyqT\nmMzJ/wCeBD6UH/8WWKcVQbWjmbPmLhivco+JmZlZ3yizSthvgX2AI4A/klZ7nUBaYO3Nbr5ukSQN\nBs4HdgXeAE6PiDMWce76+dyNgL8AB0bErwrH9wSOB0YBPwP2iYiXysRVtPCqr05MzMzM+kKZHpNj\ngAMlHQr8EPiopD8B1wI3loxjArAhsCWwH3C0pF3rT5I0HPg58Cipp+YG4AZJK+fjHyNNzD0aGAuM\noEV7+yy0T85QD+WYmZn1hR4nJhFxN2nI5obcE7EZcDtwHLB/T+uTNAz4MvD1iJgUETcBpwIHNDj9\ni8CMiNg3Ip6KiGOAJ4CP5uP7A9dExA8j4lHg88B2klbvaVz1Zs7yBn5mZmZ9rdSGLxHxfOHxn0kT\nX8sak+O4v1B2D2moqN4WwE11sYwtPN0EOKlw7DlJz+Tyv/UiRg/lmJmZLQHN7pXT9FLzEbF1D2MY\nBUzLq8vWTAWGSFqpbn7IWsCDki4CdgSeBr4REfcV6nqehU0FVuthTG9TG8oZPGgg71hu4GLONjMz\nszKa7TEp9jYMAfYAHib1cswlDaWMBS4uEcMwuu7yqak9H1xXvgJwGPA9YFtgT+DnkhQRU7qpq76e\nbg0c+PYRrpmzU2IyfNgglluuf2/gV7v+Ru1QNW6LxO3QxW2RuB0St0OXVrVBs3vlfKn2WNKlwJkR\ncWjxHEnHk+7M6anZvD1xqD1/o658HvCHiDg2P58k6VOkuSQnd1NXfT3dGj586NvK5sxLNwuPGD6Y\nESOW70l1y6xG7VBVbovE7dDFbZG4HRK3Q+uUmWOyO+kOmnoTSb0oPTUFWFnSgIiYn8tGArMiYnrd\nuS8Aj9eVPQG8t1DXyLrjI/PXNe2112bx1lvzFyp7aXrKbYYOWo5XXnm9J9UtcwYOHMDw4UMbtkPV\nuC0St0MXt0XidkjcDl1qbdFbZRKT6cAGpDVEijYjzefoqYdJw0GbALW5IpsBDzU49wFg87qydYEr\nC8fH0bUT8ntJ80se6ElAb701n3nzFn6DvfZ6mvy6wtDl3nasv2rUDlXltkjcDl3cFonbIXE7tE6Z\nxOQi4PuS1iMttjYA+Djp9t5v9bSyiJglaSJwoaTxpETiUGBvAEmrAK9GxGzgQuAASUeR1lDZG1gz\nPwa4ALhT0gM5trOAWyKiV3fkQHGfHK9hYmZm1lfKrGNyAnAKafXXm0mLqn2WdHfMeSXjOAT4HXAH\ncA7w3byeCaRhmM/k134G2IZ0R05t1dntIuKFfPwB4GukBdbuAV4CxpeMaYHOzs4F65isONS3CpuZ\nmfWVsuuYnAScJGkloDMiXu5NEBExC/hS/ld/bEDd8/vpWlCtUV0TyUM5rTJrzjzemp8mv67gNUzM\nzMz6TLPrmHyBtKLqnPy4/viCxzkx6FcWWo7eQzlmZmZ9ptkek8uB24B/0P3eM520uLeiHSycmLjH\nxMzMrK80u47JgEaPq2Lh5ejdY2JmZtZXKpdklDGjuIGfJ7+amZn1mWbnmDxNGqZZrIhYq1cRtaFa\nj8lyAwcwZJD3yTEzM+srzc4xuYImE5P+qGsNk3fQ0dGxlKMxMzPrv5qdY3JMH8fR1hYkJh7GMTMz\n61Ol1jGR9GFgfaA2rtFB2ixv44jYp0WxtY0Zs9JQju/IMTMz61s9TkwkHQJMyE87SUlJ7fFdLYqr\nrSw3IM0RfveIYUs5EjMzs/6tzF05+5OWpB8GTCPtbTMGeAy4qZuvW2btvtX72WHTNdhh0zWWdihm\nZmb9WpnEZDXgkryp3iTS8M0fSfvdfKWVwbWLUSstzy6br8WIFQcv7VDMzMz6tTKJyet0zS15Evhg\nfvwYsEYLYjIzM7OKKpOY3At8W9Iw4A/AjpIGAOOA11oZnJmZmVVLmcTkcGBb0lyTq4CRwMukPXIu\nb1lkZmZmVjk9Tkwi4lHg/cBlETETGAscC+wZEUe0OD4zMzOrkDK3C08kJSV3AkTEVODMVgdmZmZm\n1VNmgbXVgF9Ieo60VP0VEfFUa8MyMzOzKiozlLM1sDpwHrAT8KSkuySNl7RCqwM0MzOz6igz+ZWI\nmBIRp0XER0hL098FnAE838rgzMzMrFpK7ZVTI2kssBewG2lp+mtaEZSZmZlVU5nJrx8gJSOfA9YC\nfgUcBvw4Ima1NDozMzOrlDI9Jo8DT9M18fVvrQ3JzMzMqqpMYrJVRPy65ZGYmZlZ5ZW5K8dJiZmZ\nmfWJUnflmJmZmfUFJyZmZmbWNpyYmJmZWdsotY6JpKHAWxHxpqTRwPbAfRFxX0ujMzMzs0rpcY+J\npM2BKcA4SaOA3wBHAr+WtHuL4zMzM7MKKTOU89/AjcBDpEXWXgNGAl8HjmhdaGZmZlY1ZRKTDYET\nImIGsA3w04iYDdwKrNvK4MzMzKxayiQmrwODJA0BxgG35/KRwKutCszMzMyqp8zk1zuBU4FXgPnA\nbZI+Apydj5mZmZmVUqbHZH9gLvBhYK88pPN5YA5wcAtjMzMzs4rpcY9JRLwI7FZX/O2ImNuakMzM\nzKyqSi2wJmlTSSvnx58HbpB0uKSOlkZnZmZmlVJmHZOvAXcDH5b0YeByYBBpGOeolkZnZmZmlVKm\nx+Qg4L8i4g7gs8CjEfEp0jyTL7YwNjMzM6uYMonJmsAt+fEngf/Ljx8j3TJsZmZmVkqZ24X/Aawq\naS6wAfDtXD4G+HuZICQNBs4HdgXeAE6PiDMWce5NwA5AJ9CR/98hIm7Nx6cDK+Zj5OMrRsQbZWIz\nMzOzJadMYnIV8EPSQmvPAr+StAdwDvCDknFMIK0ouyWwBjBR0uSIuL7BuaNJS+HfUSh7BUDSqqSk\nZC1gVu2gkxIzM7NlQ5nE5HDgOdKH/3kR8Zak9wAXAsf0tDJJw4AvA9tExCRgkqRTgQOA6+vOHUQa\nSvptRPyjQXWjgRci4m89jcPMzMyWvjLrmMwn9Y4Uy85ZxOnNGJPjuL9Qdg+NNwQUabXZpxZR13rA\nE72IxczMzJaiMj0mSNqRlDisT1oF9s/AaRFxQ4nqRgHTImJeoWwqMETSShHxUqF8NGk34yslbUka\nSjo6Im4rHF9e0p2kJOYPwEER8ZcScZmZmdkS1uPERNKuwHXATaT5JgOAzYHrJO0aETf3sMphpOXs\ni2rPB9eVrwsMJd0JdBJpsuwtksZGxO/z8RGkCbkz8v+/lDQ6Il5vNqCBA0utO9dv1K6/6u0Abosa\nt0MXt0XidkjcDl1a1QZleky+CxwXEccWys6UdBTwHaCnicls3p6A1J4vNGk1Io6T9L2IqO1i/EdJ\nGwFfBf4T2AZ4R22yq6S9SL0qOwBXNxlPx/DhQ3t4Cf2T26GL2yJxO3RxWyRuh8Tt0Dpl0pt1SXfl\n1LuKNLTTU1OAlSUVYxkJzIqI6fUnF5KSmseAf87H5hbvwImIOcDTteNmZmbW3sokJs8DazcoXwd4\nWyLRhIdJ81Q2KZRtBjxUf6KkyyTV35L8EVJygqQnJX2hcP7yOa7HS8RlZmZmS1iZoZwfARdK2he4\nN5eNIy2Qdk1PK4uIWZIm5jrHA6sBhwJ7A0haBXg1ImaThomukvQr4D5gL+DjwFdydT8FjpX0N2Aa\ncDzwDHBries0MzOzJaxMYnIiacjmp6RVVSGtsvoTGt/i24xDSInNHcCrwHcj4qZ87AXSHjwTI+IG\nSfsBRwLvBf5EWv/k2XzuN4E3SUNN7wR+CWwfEbU4zczMrI11dHb27DNb0vsj4q+S1iUlKB3AIxHh\n4RIzMzPrlTKJyRRgl4h4sG9CMjMzs6oqM/l1bv5nZmZm1lJl5phcDtyWJ6w+SWGzPICImNiCuMzM\nzKyCygzlzO/mcGdEDOxdSGZmZlZVPU5MzMzMzPpKj4ZyJA0jrcjaWSgbDUyOiFmL/sr2J2kw6Zbl\nXUlL4Z8eEWcs3aiWnHz9vwX2j4i7ctkawMXAvwCTgYMj4hdLK8a+JGlV4GxgK9L3/1rg8Ih4s0rt\nAOnOO+A80hpBLwHnRsSEfGwNKtQWNZJ+CkyNiPH5+RpUqB0k7QxcT1oioiP//+OI+EyV2kLSIOBM\nYE/Snm6XRsR38rE1qE477A1cxsLvhw5gfkQsJ2lN4PuUbIumJ79K2jO/wIZ1h84CnpO0S7N1takJ\npGvbEtgPODpvWNjv5aTkKmC9ukM3klb63Qi4ErhB0mpLOLwl5cfAENKH8WdJ+ysdn4/dREXaQVIH\naY2iqaRVlf8TOFLSZ/MplWmLmnztn64rrtLPBqTfDTeTtgsZSdoVvrawZZXeE2cDnwA+CXwO2EfS\nPvlYldrharreByOB1UlzTs/Kx3v189FUj4mkLYH/Aa4g7W1TdCBwGHCtpC0i4r5mX7xd5J6gL5MW\na5sETJJ0KnAA6a+Efiv3eP2oQfnWwFrAJnnV3ZMlfQIYDxy3ZKPsW5IEfAxYJSKm5bKjgNMk3Qas\nCYzt7+2QrQL8Adgv78j9V0m/BMZJmkq12gJJI4BTgQcLZZX52SgYDTwaES8WC3NbVOI9kd8L44Gt\nI+J3uWwCMFbSk1SkHWDBPnT/qD2XdHh+eHgr3hPNDuUcDpwTEQc3CPBx4EuSOkkrsm7XZJ3tZAyp\nLe4vlN1D+ZVslyVbkFbIPZKFd3MeC/w+v7Fq7iF1zfU3fwe2rSUlBe8k7eFUlXYgIv5O6qYGQNLH\nSXtX7UfF2iKbAExk4Y1Aq/SzUbMe0KgrvkptMQ6YHhH31Aoi4lRY8MFclXZYSE7YvgWMj4i5knr9\nnmg2MdmAtH9Nd84nLUu/LBoFTIuIeYWyqcAQSStFxEtLKa4+FxEX1h6njoMFRpG64oqmkvYy6lfy\njtULfunm4YwDSAlbZdqhnqTJpK0ffkLqOTyLCrVF/stvM9IK1xcWDlXxPSFgW0nfAQYC1wFHUa22\nWAuYLOnzpD9aB5HmWZxItdqh3n7AlIi4IT/vdVs0m5gMoW69kgZeBoY1+8JtZhhpIlNR7fngJRxL\nu1hUm1ShPU4jJeMbk/Zxqmo77EoaP76ANOGvMu+JPO/qQtKQ1py6pL0y7QAg6X3AUNJnwO6kbvqz\nc1mV2mIF4APAV0n7t40CLiL1NFepHep9GTi58LzXbdFsYhKkbpi/dnPOpsDfmn3hNjObtzda7fkb\nVNNs4F11ZYPp5+0h6RTg68BnIuLPkirZDgAR8XsASYeQNsb8ATCi7rT+2hbHAA9FxO0NjlXqPRER\nz+Se4+m56BFJA0mTGi+jOu+JecCKwJ4R8RyApNVJPQY/B1aqO7+/tsMCkjYmDXNeUyju9c9Hs3fl\nXAkcn2+pbBTcP5PuYLi22RduM1OAlSUV22Mk6dbo6Yv4mv5uCqkNikaSdnvulySdAxwM7BURN+bi\nSrWDpPdI2qmu+M+kbusXqE5b7AHsLGmGpBnAXsB/SHoNeI7qtAMADX4PPkbqSf871WmLF4DZtaQk\nC9IQRaV+TxRsA9yVh8Nret0WzSYm55J+GP8k6XRJu0naWtLuks4C/gS8SJootix6mLT/zyaFss2A\nh5ZOOG3hAWDD3KVdMy6X9zuSjiZ10e4REdcVDlWqHUjd9NdLGlUo+yhpBv49wEYVaYstSHNLxuR/\nN5NuBx0D/IYKvSckfUrSNElDCsUbANOAu6nOe+IB0rzDtQtl65GW0XiA6rRD0Vjg3rqyXv/ObHrl\n1/wix5Nu+Sl200wldfGeuCwvsibpAtIaFuNJGfDlwN4RcdPSjGtJytsNbBkRd+Xeo0nAo6Tv+46k\nu7M+WPcXwzIv3zL9CPDfpEncRS9SkXYAyN/3+0lzxg4hJSo/IE3wO5/UTn+kAm1RJOky0pYb46v0\nswEgaQVSr9ldpNs9309aSOzM/K8y7wlJN5M+//YjzTGZSGqTC6hQO9RIeho4LCKuLZT1+uej6QXW\nImJORHyL1CWzLmlOyTrAqhFx5LKclGSHAL8D7gDOAb5bpaQkW5ClRsR8YCfS9/u3pMWEdu6nP2Q7\nkn4WjiTNJn+e1O34fG6HnalGOxS/768D95FWbzwrIs7Nx3akIm2xKBX72SAiZpK67N9N6kW+GLgw\nIk6v4HtiL9JCYneT/ng9OyLOq2A71LwHeKVY0IqfD++VY2ZmZm2j6R4TMzMzs77mxMTMzMzahhMT\nMzMzaxtOTMzMzKxtODExMzOztuHExMzMzNqGExMzMzNrG05MzMzMrG04MTEzM7O24cTErIckTZb0\nlKTlGxy7TNIdffz6T0s6qi9fo1mSNpL0mKRZkk5dxDnnSXpN0iuS3t2i191U0sdbUVc7kXSnpEtb\nWN96krbrZR3H5D1RzJYIJyZmPdcJrA6ctrQDaQNHALOB0cBJ9QclrQ/sS9qLakxEvNii172HtJmc\nde8npN2he6OTwj5aZn3NiYlZOU8BX5O09dIOZCkbATwcEZMj4pUGx99F+lD7RUQ8s2RDM6BjaQdg\n1lPLLe0AzJZRVwIfBy6V9MGIeL3RSZLmA1+MiImNyiQdDYwjbSm/PzAM+BFwAmkr9a1Jux0fGBG3\nFqpeVdKtwFaknZAnRMT5hdfYlNSDsTHwInALcHhEzMjHnwb+F9iOtGvsbhFxd4P4tyftuvwhYAZw\nFXBERMzJdbwP6JD0BWDNYvIhaW/gMlJi8pSkKyJivKTRwARg81znHcChETE1f90/kXqjPk3X7qU3\nAV+PiNm5/TqByyRtCRwLPA1sGRF35TpWL5ZJugxYHngnMBY4ISImSPo34BhgPWBKvr4TIuLNXM+n\nSdvarwfMBG4FDo6I6fVt1cz5i7v2BvUt9nxJB5J6pd5HSphPiIirC9+foyVtGRFbSxqe69sZGETa\n/fWwiPhdob6vAt8EVgVuByY3is2sr7jHxKycTuDLpB6D03tZ1+aASAnKfwFfBR4kfUhuCDxG+oAv\n+grwa2B94Azge5J2ApD0YeAXpA/FDwF75np+XlfH/sABwLbAA/VBSdqFlBDcDGyQ49ojxwVpiOAB\n4BrSFufP1lVxNbBbfrwxcKCkUaQkLHJM2wPDgfslDc3nXg6MIX14rg0cBHwhvz7AKFJPwIH5HzQe\naqgv2w34WY77Kknb5tgvJCUS+wK7AxPz9a8EXA9cQvr+7AxsBixqLk2350tatYlrL9a32PMlfYuU\nxJ4MfDBfy0RJW+TrnEJ6f+6aq/0/0jDkdsDHSN+/eyWNyfXtCZxLSl7GAPeS3idmS4x7TMxKiohn\nJX0DuEjS/0bE7SWr6gC+GhFvAE9KOg24PSJ+BCDpfGB7SasU/lK+ISJOyY/PlbQJcCgpkfgm8LPC\n8ack7QX8VdLmtV4F4NaIuLObuA4DfhwRtbkjT0raD7hR0roR8bikN4FZjeaO5F6Vl/PTaRExI3+Q\nPhsRh9TOk/RZUq9OLSn4OfDriPhTPuUZSV8nJWFExFRJAK/lOt9F4yGL+rJXIuKMwuteBVwUEZfk\nosmS9gXuyHGOIPUqPBsRzwHPSdqBRf/eXG0x5+/bxLUXNXP+gcBZEXF5PuVcSUOAd0TES5LeAmZG\nxHRJnyD1Fq1c6PE5UtK4XM94UmJ8VURclI+fKulfSEmK2RLhxMSsFyLiYkn/DlySJ3qWMTUnJTWv\nk7rka2bl/wcXyu6tq+M3pL+CIfVurC1pRt05naRJqrXE5C+LiWt90rBS0a8Lxx5fzNc3sgHwoQax\nDc6xQRrC2lHSl4B1SD0Ba5B6jnqj/no3BDaWtE+hrAOYD4yOiJ/l5OUnkl4g9UL9BLihUeURMWkx\n53d37es2qLLb83MPzSjS974Yx4RG8eX6BgDP5sSuZlD+B42/5/fhxMSWICcmZr33FeAR0pBKtyQN\nbFA8t0HZ/MVU9Vbd84HAnPx4APBDUhd/fa9BsWdjFt1r1AtRG/5tFHMzBpDmSezboP7pkjqAn5KG\nVn5EGg76PXBxD1+n0e+2+usdQBpmuaLBuS8ARMR/SDqWNN/lk6S5RXfnx2+zmPO7vfYG1S3u/HmN\nYujGAOBVUkJWX1/tvdPJ24f4y36vzUrxHBOzXoqIZ4FvkOacbFZ3eC5pXkDNB1r0shvVPd8M+GN+\n/CiwXkQ8HRFPRcRTpL+IzwLe24PXeIQ076Voc9KH1597HvKC2EYDzxViewX4Humv9Y+Q5rz8e0Qc\nERFXkXqP1mbRd5i8mf+vb+fF3eL6KKBaHDmW95HmV6wo6WOSzoiIv0TE2RGxA2m4Y2tJK9dX1sT5\ni7v2HrVVRLxGmhi9cV0c10mq9Zp01tU3HBhcd82HAzvlcx4mTeou2hizJcg9JmYtEBE/kLQ78Cmg\neFvs/cA+ku4m/SFwBmndj56q/1DeU9IjpKGCXUgfLFvlY6cDd0k6lzSRcQRwHjAEeKIHr3kqcK2k\n7wDXkiZ0ngPcEhE9qacY+/mkSaw/lFTr0ZlAmqT7KCmBmgvsIWkasDJprZRVWHgoayYwWtK7IuIF\nSZOBgyT9JX/N8Sw+MTkFuEbSd0k9M+8jTVx9MiL+keeu7J/n0VwMDCVN/n0iIqY1qO+17s7Pc4W6\nu/Z6zZx/MnCipCdI77V/A3YEPlFop3UkvQe4DZiUr/lA0mTl/YG9ST1stfpuynOnbiT1/OxGSoDM\nlgj3mJj13KI+8L5C6mIvHt8XeJn0oXEd8H3guRL1d9Y9Po10l8Yk4IvAnrXbfSPiN8A2pHkBvyN9\nwDwG/GtEzGtQX0MRcT3pjp7dSb0n55M+wPZY3NcuKvaImAxsAaxIWiTtTtIQy1YR8VJEvED6oNyR\n1CtzLam9zmThhcJOJ03UrK2S+nnSrcAPk+5M+TaLGQ6LiB/na9k5X99E0l0ru+Xjj5OSvq2AP5CG\nZObRNZenvr5uz1/ctTeob7HnR8S5pCTsOFKyMh74TETck6s5m5Ss/Cwi5gP/SrpF+BrSe2ccsHNE\n/CrXdyvwuVzPI7ltFjVnxaxPdHR2ekE/MzMzaw/uMTEzM7O24cTEzMzM2oYTEzMzM2sbTkzMzMys\nbTgxMTMzs7bhxMTMzMzahhMTMzMzaxtOTMzMzKxtODExMzOztuHExMzMzNqGExMzMzNrG/8PWtIy\nV+ijP7UAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x27094fa90>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "cv = model_selection.KFold(shuffle=True, n_splits=10)\n",
    "rfecv = feature_selection.RFECV(linear_model.LogisticRegression(), cv=cv)\n",
    "rfecv.fit(X, y)\n",
    "cols = rfecv.get_support()\n",
    "X2 = X[X.columns[cols]]\n",
    "\n",
    "plt.figure()\n",
    "plt.xlabel(\"Number of features selected\")\n",
    "plt.ylabel(\"Cross validation score (nb of correct classifications)\")\n",
    "plt.plot(range(1, len(rfecv.grid_scores_) + 1), rfecv.grid_scores_)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Application"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 333,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean cross-val accuracy score:  0.825248392753 +/- 0.0425540776049\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.84      0.82      0.83       150\n",
      "          1       0.81      0.83      0.82       142\n",
      "\n",
      "avg / total       0.83      0.83      0.83       292\n",
      "\n",
      "AUC: 0.894014084507\n"
     ]
    }
   ],
   "source": [
    "model_summary(linear_model.LogisticRegressionCV(), X2, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 334,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean cross-val accuracy score:  0.821858562244 +/- 0.0444525231664\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.83      0.85      0.84       150\n",
      "          1       0.83      0.82      0.83       142\n",
      "\n",
      "avg / total       0.83      0.83      0.83       292\n",
      "\n",
      "AUC: 0.883685446009\n"
     ]
    }
   ],
   "source": [
    "model_summary(linear_model.LogisticRegressionCV(penalty='l1', solver='liblinear'), X2, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 335,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean cross-val accuracy score:  0.821858562244 +/- 0.0387350301281\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.81      0.85      0.83       150\n",
      "          1       0.83      0.79      0.81       142\n",
      "\n",
      "avg / total       0.82      0.82      0.82       292\n",
      "\n",
      "AUC: 0.863028169014\n"
     ]
    }
   ],
   "source": [
    "model_summary(ensemble.RandomForestClassifier(), X2, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 336,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean cross-val accuracy score:  0.82162478083 +/- 0.0567945110531\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.82      0.85      0.83       150\n",
      "          1       0.84      0.80      0.82       142\n",
      "\n",
      "avg / total       0.83      0.83      0.83       292\n",
      "\n",
      "AUC: 0.887840375587\n"
     ]
    }
   ],
   "source": [
    "model_summary(xgboost.XGBClassifier(), X2, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 313,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean cross-val accuracy score:  0.818850574713 +/- 0.0601912660255\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.80      0.84      0.82       150\n",
      "          1       0.82      0.78      0.80       142\n",
      "\n",
      "avg / total       0.81      0.81      0.81       292\n",
      "\n",
      "AUC: 0.879741784038\n"
     ]
    }
   ],
   "source": [
    "model_summary(naive_bayes.BernoulliNB(), X2, y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Logistic Regression is the best model, plus it allows for better interpretation, so I will use that model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 384,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy: 0.835616438356\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>actual</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cross_val_pred</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>128</td>\n",
       "      <td>26</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>22</td>\n",
       "      <td>116</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "actual            0    1\n",
       "cross_val_pred          \n",
       "0               128   26\n",
       "1                22  116"
      ]
     },
     "execution_count": 384,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# confusion matrix\n",
    "predictions = model_selection.cross_val_predict(linear_model.LogisticRegressionCV(), X2, y)\n",
    "actual_vs_cross_val_pred = pd.DataFrame([predictions, list(y.values)]).transpose()\n",
    "actual_vs_cross_val_pred.columns = ['cross_val_pred', 'actual']\n",
    "print 'accuracy:', metrics.accuracy_score(predictions, y)\n",
    "pd.crosstab(actual_vs_cross_val_pred.cross_val_pred, actual_vs_cross_val_pred.actual)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Interpretation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 358,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "features = list(X2.columns.values)\n",
    "features.append('intercept')\n",
    "\n",
    "coefficients = list(log_regCV.coef_[0,:].round(3))\n",
    "coefficients.append(log_regCV.intercept_[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 380,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda/envs/py27/lib/python2.7/site-packages/ipykernel/__main__.py:20: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>*features</th>\n",
       "      <th>coefficients</th>\n",
       "      <th>odds_ratio</th>\n",
       "      <th>interpretation</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>associate_in_title</td>\n",
       "      <td>-2.519000</td>\n",
       "      <td>0.081</td>\n",
       "      <td>1134.57% lower odds</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>intercept</td>\n",
       "      <td>-2.476641</td>\n",
       "      <td>0.084</td>\n",
       "      <td>Base Odds: 1 to 11.9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>risk_in_title</td>\n",
       "      <td>-1.695000</td>\n",
       "      <td>0.184</td>\n",
       "      <td>443.48% lower odds</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>assistant_in_title</td>\n",
       "      <td>-1.500000</td>\n",
       "      <td>0.223</td>\n",
       "      <td>348.43% lower odds</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>analyst_in_title</td>\n",
       "      <td>-1.298000</td>\n",
       "      <td>0.273</td>\n",
       "      <td>266.3% lower odds</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>research_in_title</td>\n",
       "      <td>-1.017000</td>\n",
       "      <td>0.362</td>\n",
       "      <td>176.24% lower odds</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>search_city_Seattle, WA</td>\n",
       "      <td>-0.716000</td>\n",
       "      <td>0.489</td>\n",
       "      <td>104.5% lower odds</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>scientist_in_summary</td>\n",
       "      <td>0.981000</td>\n",
       "      <td>2.667</td>\n",
       "      <td>166.7% greater odds</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>looking_in_summary</td>\n",
       "      <td>1.292000</td>\n",
       "      <td>3.640</td>\n",
       "      <td>264.0% greater odds</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>science_in_title</td>\n",
       "      <td>1.302000</td>\n",
       "      <td>3.677</td>\n",
       "      <td>267.7% greater odds</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ds_in_name</td>\n",
       "      <td>1.368000</td>\n",
       "      <td>3.927</td>\n",
       "      <td>292.7% greater odds</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>machine_in_title</td>\n",
       "      <td>1.471000</td>\n",
       "      <td>4.354</td>\n",
       "      <td>335.4% greater odds</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>expensive_city</td>\n",
       "      <td>1.476000</td>\n",
       "      <td>4.375</td>\n",
       "      <td>337.5% greater odds</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>learning_in_title</td>\n",
       "      <td>1.479000</td>\n",
       "      <td>4.389</td>\n",
       "      <td>338.9% greater odds</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>engineer_in_title</td>\n",
       "      <td>1.588000</td>\n",
       "      <td>4.894</td>\n",
       "      <td>389.4% greater odds</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>in_city</td>\n",
       "      <td>1.612000</td>\n",
       "      <td>5.013</td>\n",
       "      <td>401.3% greater odds</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>software_in_title</td>\n",
       "      <td>1.717000</td>\n",
       "      <td>5.568</td>\n",
       "      <td>456.8% greater odds</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>quantitative_in_title</td>\n",
       "      <td>2.453000</td>\n",
       "      <td>11.623</td>\n",
       "      <td>1062.3% greater odds</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  *features  coefficients  odds_ratio        interpretation\n",
       "7        associate_in_title     -2.519000       0.081   1134.57% lower odds\n",
       "17                intercept     -2.476641       0.084  Base Odds: 1 to 11.9\n",
       "13            risk_in_title     -1.695000       0.184    443.48% lower odds\n",
       "6        assistant_in_title     -1.500000       0.223    348.43% lower odds\n",
       "5          analyst_in_title     -1.298000       0.273     266.3% lower odds\n",
       "12        research_in_title     -1.017000       0.362    176.24% lower odds\n",
       "16  search_city_Seattle, WA     -0.716000       0.489     104.5% lower odds\n",
       "4      scientist_in_summary      0.981000       2.667   166.7% greater odds\n",
       "3        looking_in_summary      1.292000       3.640   264.0% greater odds\n",
       "14         science_in_title      1.302000       3.677   267.7% greater odds\n",
       "2                ds_in_name      1.368000       3.927   292.7% greater odds\n",
       "10         machine_in_title      1.471000       4.354   335.4% greater odds\n",
       "1            expensive_city      1.476000       4.375   337.5% greater odds\n",
       "9         learning_in_title      1.479000       4.389   338.9% greater odds\n",
       "8         engineer_in_title      1.588000       4.894   389.4% greater odds\n",
       "0                   in_city      1.612000       5.013   401.3% greater odds\n",
       "15        software_in_title      1.717000       5.568   456.8% greater odds\n",
       "11    quantitative_in_title      2.453000      11.623  1062.3% greater odds"
      ]
     },
     "execution_count": 380,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "log_regCV = linear_model.LogisticRegressionCV()\n",
    "log_regCV.fit(X2, y)\n",
    "\n",
    "df = pd.DataFrame({'*features' : features, \n",
    "                   'coefficients': coefficients})\n",
    "df['odds_ratio'] = (math.e**df.coefficients).round(3)\n",
    "def interpretation(odds_ratio):\n",
    "    if odds_ratio == 1.0:\n",
    "        return \"no change\"\n",
    "    elif odds_ratio < 1:\n",
    "        inv = odds_ratio**-1\n",
    "        pct = round((100*(inv-1)), 2)\n",
    "        return str(pct)+'% lower odds'\n",
    "    else:\n",
    "        pct = round((100*(odds_ratio-1)), 2)\n",
    "        return str(pct)+'% greater odds'\n",
    "df['interpretation'] = df.odds_ratio.apply(interpretation)\n",
    "\n",
    "odds = np.e**log_regCV.intercept_[0]\n",
    "df.interpretation[df['*features'] == 'intercept'] = 'Base Odds: 1 to ' + str((odds**-1).round(2))\n",
    "\n",
    "df.sort_values('coefficients')"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda env:py27]",
   "language": "python",
   "name": "conda-env-py27-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}