pra1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "1ee8b497",
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "import time\n",
    "import pandas as pd\n",
    "import selenium\n",
    "from bs4 import BeautifulSoup\n",
    "from dateutil import parser\n",
    "from selenium.webdriver import Chrome\n",
    "from selenium.webdriver.chrome.options import Options\n",
    "from webdriver_manager.chrome import ChromeDriverManager\n",
    "import datetime\n",
    "import requests"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "fe7a754d",
   "metadata": {},
   "outputs": [],
   "source": [
    "def scrapeMatch(url,date,browser):\n",
    "    browser.get(url)\n",
    "    html = browser.page_source\n",
    "    bs = BeautifulSoup(html, 'html.parser')\n",
    "    tables = bs.find_all('table')\n",
    "    #for table in tables:\n",
    "    final_columns = ['DATE','DATE_2','TEAM_HOME', 'TEAM_VISITOR', 'POINTS_HOME', 'POINTS_VISITOR']\n",
    "    df_list = []\n",
    "    tableList = []\n",
    "    columns = ['DATE']\n",
    "    for th in tables[0].find('thead').findAll('th'):\n",
    "        text = th.text\n",
    "        if text == '\\xa0':\n",
    "            text = 'TEAM'\n",
    "        columns.append(text)\n",
    "    for tr in tables[0].find('tbody').find_all('tr'):\n",
    "        row = [date]\n",
    "        for child in tr.findChildren(recursive=False):\n",
    "            row.append(child.text)\n",
    "        tableList.append(row)\n",
    "    df = pd.DataFrame(columns=columns, data=tableList)\n",
    "    df.drop(df.columns.difference(['DATE','TEAM','FINAL']), 1, inplace=True)\n",
    "    df2 = df.unstack().to_frame().T\n",
    "    df2.columns = final_columns\n",
    "    df3=df2.loc[:, df2.columns.drop(['DATE_2'])]\n",
    "    return df3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "bfb8ed73",
   "metadata": {},
   "outputs": [],
   "source": [
    "def check_winner(el):\n",
    "    if el['POINTS_VISITOR'] != '' and el['POINTS_HOME'] != '':\n",
    "        if el['POINTS_VISITOR'] > el['POINTS_HOME']:\n",
    "            return el['TEAM_VISITOR']\n",
    "        else:\n",
    "            return el['TEAM_HOME']\n",
    "    else:\n",
    "        return None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "0b7dbc5c",
   "metadata": {},
   "outputs": [],
   "source": [
    "def daterange(start_date, end_date):\n",
    "    for n in range(int((end_date - start_date).days)):\n",
    "        yield start_date + timedelta(n)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "a3f7aefa",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "====== WebDriver manager ======\n",
      "Current google-chrome version is 100.0.4896\n",
      "Get LATEST chromedriver version for 100.0.4896 google-chrome\n",
      "There is no [win32] chromedriver for browser 100.0.4896 in cache\n",
      "Trying to download new driver from https://chromedriver.storage.googleapis.com/100.0.4896.60/chromedriver_win32.zip\n",
      "Driver has been saved in cache [C:\\Users\\Francis Fernández\\.wdm\\drivers\\chromedriver\\win32\\100.0.4896.60]\n",
      "C:\\Users\\FRANCI~1\\AppData\\Local\\Temp/ipykernel_8404/3034120236.py:8: DeprecationWarning: executable_path has been deprecated, please pass in a Service object\n",
      "  with Chrome(ChromeDriverManager().install(),options=chrome_options) as browser:\n",
      "C:\\Users\\FRANCI~1\\AppData\\Local\\Temp/ipykernel_8404/1642095102.py:22: FutureWarning: In a future version of pandas all arguments of DataFrame.drop except for the argument 'labels' will be keyword-only\n",
      "  df.drop(df.columns.difference(['DATE','TEAM','FINAL']), 1, inplace=True)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>DATE</th>\n",
       "      <th>TEAM_HOME</th>\n",
       "      <th>TEAM_VISITOR</th>\n",
       "      <th>POINTS_HOME</th>\n",
       "      <th>POINTS_VISITOR</th>\n",
       "      <th>WINNER</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2022-03-01</td>\n",
       "      <td>DET</td>\n",
       "      <td>WAS</td>\n",
       "      <td>113</td>\n",
       "      <td>116</td>\n",
       "      <td>WAS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2022-03-01</td>\n",
       "      <td>LAC</td>\n",
       "      <td>HOU</td>\n",
       "      <td>113</td>\n",
       "      <td>100</td>\n",
       "      <td>LAC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2022-03-01</td>\n",
       "      <td>GSW</td>\n",
       "      <td>MIN</td>\n",
       "      <td>114</td>\n",
       "      <td>129</td>\n",
       "      <td>MIN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2022-03-01</td>\n",
       "      <td>BKN</td>\n",
       "      <td>TOR</td>\n",
       "      <td>108</td>\n",
       "      <td>109</td>\n",
       "      <td>TOR</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2022-03-01</td>\n",
       "      <td>DAL</td>\n",
       "      <td>LAL</td>\n",
       "      <td>109</td>\n",
       "      <td>104</td>\n",
       "      <td>DAL</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2022-03-28</td>\n",
       "      <td>SAC</td>\n",
       "      <td>MIA</td>\n",
       "      <td>100</td>\n",
       "      <td>123</td>\n",
       "      <td>MIA</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2022-03-28</td>\n",
       "      <td>ATL</td>\n",
       "      <td>IND</td>\n",
       "      <td>132</td>\n",
       "      <td>123</td>\n",
       "      <td>ATL</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2022-03-28</td>\n",
       "      <td>GSW</td>\n",
       "      <td>MEM</td>\n",
       "      <td>95</td>\n",
       "      <td>123</td>\n",
       "      <td>GSW</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2022-03-28</td>\n",
       "      <td>DEN</td>\n",
       "      <td>CHA</td>\n",
       "      <td>113</td>\n",
       "      <td>109</td>\n",
       "      <td>DEN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2022-03-28</td>\n",
       "      <td>OKC</td>\n",
       "      <td>POR</td>\n",
       "      <td>134</td>\n",
       "      <td>131</td>\n",
       "      <td>OKC</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>208 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          DATE TEAM_HOME TEAM_VISITOR POINTS_HOME POINTS_VISITOR WINNER\n",
       "0   2022-03-01       DET          WAS         113            116    WAS\n",
       "0   2022-03-01       LAC          HOU         113            100    LAC\n",
       "0   2022-03-01       GSW          MIN         114            129    MIN\n",
       "0   2022-03-01       BKN          TOR         108            109    TOR\n",
       "0   2022-03-01       DAL          LAL         109            104    DAL\n",
       "..         ...       ...          ...         ...            ...    ...\n",
       "0   2022-03-28       SAC          MIA         100            123    MIA\n",
       "0   2022-03-28       ATL          IND         132            123    ATL\n",
       "0   2022-03-28       GSW          MEM          95            123    GSW\n",
       "0   2022-03-28       DEN          CHA         113            109    DEN\n",
       "0   2022-03-28       OKC          POR         134            131    OKC\n",
       "\n",
       "[208 rows x 6 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "chrome_options = Options()  \n",
    "chrome_options.add_argument(\"--headless\")\n",
    "chrome_options.add_argument('log-level=3')\n",
    "start_date = datetime.date(2022, 3, 1)\n",
    "end_date = datetime.date(2022, 3, 28)\n",
    "delta = datetime.timedelta(days=1)\n",
    "df_list = []\n",
    "with Chrome(ChromeDriverManager().install(),options=chrome_options) as browser:\n",
    "    while (start_date <= end_date):\n",
    "        date = start_date\n",
    "        start_date += delta\n",
    "        url = 'https://www.nba.com/games?date='+ date.strftime(\"%Y-%m-%d\")\n",
    "        response = requests.get(url)\n",
    "        if response.status_code != 200:\n",
    "            pass\n",
    "        browser.get(url)\n",
    "        html = browser.page_source\n",
    "        bs = BeautifulSoup(html, 'html.parser')\n",
    "        links = bs.find_all('a', {'href': re.compile(r'/game/[a-z-]+vs[a-z-0-9]+$')})\n",
    "        refs = [link['href'] for link in links]\n",
    "        refs = set(refs)\n",
    "        for ref in refs:\n",
    "            url = 'https://www.nba.com' + ref\n",
    "            df_list.append(scrapeMatch(url,date,browser))\n",
    "    df_final = pd.concat(df_list)\n",
    "    df_final['WINNER'] = df_final.apply (lambda row: check_winner(row), axis=1)\n",
    "    display(df_final)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "7a12a501",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_final.to_csv('dataset_pra1.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1323cf35",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}