crawling_post.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%pip install selenium"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                                title  \\\n",
      "0                취약계층 위기처소년 자립현황 및 대응방안모색 현장포럼 및 업무협약   \n",
      "1                    2023 경기도 자립두배통장 안내& 자주하는 질문!! 모음   \n",
      "2                     (홍보자료) 경기도 청소년 자립두배통장 참가자 모집 안내   \n",
      "3            (홍보자료)'청소년복지시설 퇴소 경계선지능 청소년 주거지원사업' 홍보영상   \n",
      "4                    (홍보자료) 경기북부청소년자립지원관 특별지원 택배발송 안내   \n",
      "5             '청소년복지시설 퇴소 경계선지능 청소년 주거지원사업' 자립지원 매뉴얼북   \n",
      "6   (홍보자료) '청소년복지시설 퇴소 경계선지능 청소년 주거지원사업' 성과연구 결과공유...   \n",
      "7                          2022년 경기북부청소년자립지원관 하반기 소식지   \n",
      "8                        (홍보자료) 경기북부청소년자립지원관 10월 카드뉴스   \n",
      "9                          (홍보자료) 경기북부청소년자립지원관 개관 4주년   \n",
      "10                          (홍보자료)마스크팩 2종, 틴트 후원물품 수령   \n",
      "11  (홍보자료) 2022 경기도 가정 밖 청소년 자립 지원 마라톤 '너를 응RUN해' ...   \n",
      "12                   (홍보자료) 경기북부청소년자립지원관 9월 비전모임 카드뉴스   \n",
      "13                               (홍보자료) '섬유향수'후원물품 수령   \n",
      "14                        (홍보자료) 경기북부청소년자립지원관 8월 카드뉴스   \n",
      "15                            [FORTEN 3월 4주차 아웃리치 공지]   \n",
      "16                            [FORTEN 3월 3주차 아웃리치 공지]   \n",
      "17                            [FORTEN 3월 2주차 아웃리치 공지]   \n",
      "18                            [FORTEN 3월 1주차 아웃리치 공지]   \n",
      "19                                             긴급생계지원   \n",
      "20                            [FORTEN 2월 4주차 아웃리치 공지]   \n",
      "21                            [FORTEN 2월 3주차 아웃리치 공지]   \n",
      "22                           [2023년 FORTEN 아웃리치 OPEN]   \n",
      "23                                      [FORTEN 휴무공지]   \n",
      "24                                                      \n",
      "25  새빛과 빛쟁이의 첫 출사너무 멋진 꿈드림 동아리의 B컷 사진친구들의 다양한 활동 많...   \n",
      "26                                   [3월 꿈공작소 참여자 모집]   \n",
      "27                                  안녕하세요^^의정부시꿈드림입니다   \n",
      "28  #의정부시학교밖청소년지원센터 #의정부시꿈드림 #꿈공작소 #의정부시청소년상담복지센터 ...   \n",
      "29                          의정부시꿈드림에는 숨은 고수 친구들이 많답니다   \n",
      "30                       의정부시꿈드림의 특화프로그램인 꿈공작소를 안내합니다   \n",
      "31  ★학교 밖 우리 사회 곳곳의 목소리를 담아 뜻있는 콘텐츠를 직접 만들고 싶은 청소년...   \n",
      "32                                 [찰리채플린 콘서트 참가자 모집]   \n",
      "33   경기도학교밖청소년지원센터에서 경기도 학교밖청소년을 대표하는 꿈드림단 청소년을 모집합니다   \n",
      "34                         2023년도 의정부꿈드림 사업설명회가 진행됩니다   \n",
      "\n",
      "                                                  URL  \\\n",
      "0   http://www.stand1318.com/bbs/board.php?bo_tabl...   \n",
      "1   http://www.stand1318.com/bbs/board.php?bo_tabl...   \n",
      "2   http://www.stand1318.com/bbs/board.php?bo_tabl...   \n",
      "3   http://www.stand1318.com/bbs/board.php?bo_tabl...   \n",
      "4   http://www.stand1318.com/bbs/board.php?bo_tabl...   \n",
      "5   http://www.stand1318.com/bbs/board.php?bo_tabl...   \n",
      "6   http://www.stand1318.com/bbs/board.php?bo_tabl...   \n",
      "7   http://www.stand1318.com/bbs/board.php?bo_tabl...   \n",
      "8   http://www.stand1318.com/bbs/board.php?bo_tabl...   \n",
      "9   http://www.stand1318.com/bbs/board.php?bo_tabl...   \n",
      "10  http://www.stand1318.com/bbs/board.php?bo_tabl...   \n",
      "11  http://www.stand1318.com/bbs/board.php?bo_tabl...   \n",
      "12  http://www.stand1318.com/bbs/board.php?bo_tabl...   \n",
      "13  http://www.stand1318.com/bbs/board.php?bo_tabl...   \n",
      "14  http://www.stand1318.com/bbs/board.php?bo_tabl...   \n",
      "15  https://www.facebook.com/forten1318/posts/pfbi...   \n",
      "16  https://www.facebook.com/forten1318/posts/pfbi...   \n",
      "17  https://www.facebook.com/forten1318/posts/pfbi...   \n",
      "18  https://www.facebook.com/forten1318/posts/pfbi...   \n",
      "19  https://www.facebook.com/forten1318/posts/pfbi...   \n",
      "20  https://www.facebook.com/forten1318/posts/pfbi...   \n",
      "21  https://www.facebook.com/forten1318/posts/pfbi...   \n",
      "22  https://www.facebook.com/forten1318/posts/pfbi...   \n",
      "23  https://www.facebook.com/forten1318/posts/pfbi...   \n",
      "24  https://www.facebook.com/forten1318/posts/pfbi...   \n",
      "25  https://www.facebook.com/dreams9570/posts/pfbi...   \n",
      "26  https://www.facebook.com/dreams9570/posts/pfbi...   \n",
      "27  https://www.facebook.com/dreams9570/posts/pfbi...   \n",
      "28  https://www.facebook.com/dreams9570/posts/pfbi...   \n",
      "29  https://www.facebook.com/dreams9570/posts/pfbi...   \n",
      "30  https://www.facebook.com/dreams9570/posts/pfbi...   \n",
      "31  https://www.facebook.com/dreams9570/posts/pfbi...   \n",
      "32  https://www.facebook.com/dreams9570/posts/pfbi...   \n",
      "33  https://www.facebook.com/dreams9570/posts/pfbi...   \n",
      "34  https://www.facebook.com/dreams9570/posts/pfbi...   \n",
      "\n",
      "                                                  Img  \n",
      "0   http://www.stand1318.com/data/editor/2303/thum...  \n",
      "1   http://www.stand1318.com/data/editor/2302/thum...  \n",
      "2   http://www.stand1318.com/data/editor/2302/thum...  \n",
      "3   http://www.stand1318.com/data/editor/2301/thum...  \n",
      "4   http://www.stand1318.com/data/editor/2301/thum...  \n",
      "5   http://www.stand1318.com/data/editor/2301/thum...  \n",
      "6   http://www.stand1318.com/data/editor/2301/thum...  \n",
      "7   http://www.stand1318.com/data/editor/2301/thum...  \n",
      "8   http://www.stand1318.com/data/editor/2211/thum...  \n",
      "9   http://www.stand1318.com/data/editor/2211/thum...  \n",
      "10  http://www.stand1318.com/data/editor/2210/thum...  \n",
      "11  http://www.stand1318.com/data/editor/2210/thum...  \n",
      "12  http://www.stand1318.com/data/editor/2210/thum...  \n",
      "13  http://www.stand1318.com/data/editor/2209/thum...  \n",
      "14  http://www.stand1318.com/data/editor/2209/thum...  \n",
      "15  https://scontent-gmp1-1.xx.fbcdn.net/v/t39.308...  \n",
      "16  https://scontent-gmp1-1.xx.fbcdn.net/v/t39.308...  \n",
      "17  https://scontent-gmp1-1.xx.fbcdn.net/v/t39.308...  \n",
      "18  https://scontent-gmp1-1.xx.fbcdn.net/v/t39.308...  \n",
      "19  https://scontent-gmp1-1.xx.fbcdn.net/v/t39.308...  \n",
      "20  https://scontent-gmp1-1.xx.fbcdn.net/v/t39.308...  \n",
      "21  https://scontent-gmp1-1.xx.fbcdn.net/v/t39.308...  \n",
      "22  https://scontent-gmp1-1.xx.fbcdn.net/v/t39.308...  \n",
      "23  https://scontent-gmp1-1.xx.fbcdn.net/v/t39.308...  \n",
      "24  https://scontent-gmp1-1.xx.fbcdn.net/v/t39.308...  \n",
      "25  https://scontent-gmp1-1.xx.fbcdn.net/v/t39.308...  \n",
      "26  https://scontent-gmp1-1.xx.fbcdn.net/v/t39.308...  \n",
      "27  https://scontent-gmp1-1.xx.fbcdn.net/v/t39.308...  \n",
      "28  https://scontent-gmp1-1.xx.fbcdn.net/v/t39.308...  \n",
      "29  https://scontent-gmp1-1.xx.fbcdn.net/v/t39.308...  \n",
      "30  https://scontent-gmp1-1.xx.fbcdn.net/v/t39.308...  \n",
      "31  https://scontent-gmp1-1.xx.fbcdn.net/v/t39.308...  \n",
      "32  https://scontent-gmp1-1.xx.fbcdn.net/v/t39.308...  \n",
      "33  https://scontent-gmp1-1.xx.fbcdn.net/v/t39.308...  \n",
      "34  https://scontent-gmp1-1.xx.fbcdn.net/v/t39.308...  \n"
     ]
    }
   ],
   "source": [
    "from selenium import webdriver\n",
    "from bs4 import BeautifulSoup\n",
    "import pandas as pd\n",
    "import time\n",
    "import os\n",
    "from dotenv import load_dotenv\n",
    "from selenium.webdriver.common.by import By\n",
    "import re\n",
    "\n",
    "urls=[\"http://www.stand1318.com/bbs/board.php?bo_table=5_2\",\"https://www.facebook.com/forten1318?locale=ko_KR\",\"https://www.facebook.com/dreams9570/\"]\n",
    "\n",
    "# Create a new instance of the Firefox driver\n",
    "driver = webdriver.Chrome()\n",
    "\n",
    "\n",
    "post_list=[]\n",
    "\n",
    "for i in range(len(urls)):\n",
    "    if \"facebook\" in urls[i]:\n",
    "        # Load the environment variables from the .env file\n",
    "      \n",
    "        load_dotenv()\n",
    "        email = os.getenv('EMAIL')\n",
    "        password = os.getenv('PASSWORD')\n",
    "\n",
    "        # Set up the Selenium webdriver\n",
    "        driver.get('https://www.facebook.com')\n",
    "\n",
    "        # Find the email and password fields, and enter your login credentials\n",
    "        email_field = driver.find_element(By.ID,'email')\n",
    "        email_field.send_keys(email)\n",
    "\n",
    "        password_field = driver.find_element(By.ID,'pass')\n",
    "        password_field.send_keys(password)\n",
    "\n",
    "        # Find the login button and click it\n",
    "        login_button = driver.find_element(By.NAME,'login')\n",
    "        login_button.click()\n",
    "\n",
    "        # Wait for the page to load and navigate to the target page\n",
    "        time.sleep(5)\n",
    "        driver.get(urls[i])\n",
    "\n",
    "\n",
    "        while True:\n",
    "            driver.execute_script('window.scrollTo(0, document.body.scrollHeight);')\n",
    "            time.sleep(5)\n",
    "            page_source = driver.page_source\n",
    "            soup = BeautifulSoup(page_source, 'html.parser')\n",
    "            posts = soup.find_all('div', {'class': 'x1yztbdb x1n2onr6 xh8yej3 x1ja2u2z'})\n",
    "            if len(posts) >= 5:\n",
    "                break\n",
    "        \n",
    "        # Print the post text and url\n",
    "        for post in posts:\n",
    "            post_text = post.find('div', {'class': 'x11i5rnm xat24cr x1mh8g0r x1vvkbs xdj266r x126k92a'})\n",
    "            post_url = post.find('a', {'class': 'x1i10hfl xjbqb8w x6umtig x1b1mbwd xaqea5y xav7gou x9f619 x1ypdohk xt0psk2 xe8uvvx xdj266r x11i5rnm xat24cr x1mh8g0r xexx8yu x4uap5 x18d9i69 xkhd6sd x16tdsg8 x1hl2dhg xggy1nq x1a2a7pz x1heor9g xt0b8zv xo1l8bm'})\n",
    "            text = post_text.text.strip()\n",
    "            first_sentence = re.split('[._\\-\\]]\\s*',text)[0]\n",
    "            if '[' in first_sentence:\n",
    "                first_sentence = first_sentence + ']'\n",
    "\n",
    "            if post_text is not None and post_url is not None:\n",
    "                #print(first_sentence)\n",
    "                #print(post_url['href'])\n",
    "                if urls[i] == \"https://www.facebook.com/forten1318?locale=ko_KR\":\n",
    "                    image_src = \"https://scontent-gmp1-1.xx.fbcdn.net/v/t39.30808-6/327196562_1601959513589572_1287926740667880922_n.png?_nc_cat=103&ccb=1-7&_nc_sid=09cbfe&_nc_ohc=Qb--TgSn4mIAX88LAGe&_nc_ht=scontent-gmp1-1.xx&oh=00_AfDZdLYEy4RZwOZJhV2ZpUlHM2HpPp-uPPPDKk2ezZWWXA&oe=6425852E\"\n",
    "                else:\n",
    "                    image_src = \"https://scontent-gmp1-1.xx.fbcdn.net/v/t39.30808-6/307290627_460262782798023_4295563063071358716_n.jpg?_nc_cat=101&ccb=1-7&_nc_sid=09cbfe&_nc_ohc=2KbSjNfdHuUAX-NEtNw&_nc_ht=scontent-gmp1-1.xx&oh=00_AfCFZAC5oUZWODCXjOOSd2kmJpRlIx6m_hiECRoGzFmjuw&oe=64259056\"\n",
    "                \n",
    "                post_item ={\n",
    "                'title': first_sentence,\n",
    "                'URL': post_url['href'],\n",
    "                'Img': image_src\n",
    "                }\n",
    "                post_list.append(post_item)\n",
    "                \n",
    "\n",
    "    else:\n",
    "        # Navigate to the website\n",
    "        driver.get(urls[i])\n",
    "        # Wait for the page to load\n",
    "        driver.implicitly_wait(10)\n",
    "\n",
    "        # Get the HTML source of the page\n",
    "        html = driver.page_source\n",
    "\n",
    "        # Parse the HTML with BeautifulSoup\n",
    "        soup = BeautifulSoup(html, 'html.parser')\n",
    "\n",
    "        # Find all the posts\n",
    "        posts = soup.find_all('div', class_='gall_con')\n",
    "\n",
    "        # Loop through each post and extract the title and date\n",
    "        for post in posts:\n",
    "            # Find the title and date elements\n",
    "            title_elem = post.find('a', class_='bo_tit')\n",
    "            date_elem = post.find('span', class_='gall_date')\n",
    "            url_elem = post.find('a',class_='bo_tit')\n",
    "            img_elem = post.find('img')\n",
    "\n",
    "            \n",
    "            # Extract the title and date text\n",
    "            title = title_elem.text.strip()\n",
    "            date_text = date_elem.text.strip()\n",
    "            date_str = date_text.split(' ')[-1]\n",
    "\n",
    "\n",
    "            post_item ={\n",
    "                'title': title,\n",
    "                'URL': url_elem['href'],\n",
    "                'Img': img_elem['src']\n",
    "            }\n",
    "            post_list.append(post_item)\n",
    "\n",
    "\n",
    "df = pd.DataFrame(post_list)\n",
    "print(df)\n",
    "\n",
    "# Close the browser\n",
    "driver.quit()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "import firebase_admin\n",
    "from firebase_admin import credentials\n",
    "from firebase_admin import firestore\n",
    "\n",
    "\n",
    "cred = credentials.Certificate(\"teenhelper-93c4e-firebase-adminsdk-293wr-2bd24cb784.json\")\n",
    "firebase_admin.initialize_app(cred)\n",
    "db = firestore.client()\n",
    "\n",
    "data_dict = df.to_dict('list')\n",
    "df_notification = df.drop(['URL'],axis=1,inplace=True)\n",
    "\n",
    "def save(collection_id, document_id, data):\n",
    "    db.collection(collection_id).document(str(document_id)).set(data)\n",
    "\n",
    "\n",
    "for i,row in df.iterrows():\n",
    "    save(\n",
    "        collection_id=\"Notification\", \n",
    "        document_id=i,\n",
    "        data=row.to_dict()\n",
    "    )\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from datetime import datetime\n",
    "from pytz import timezone\n",
    "\n",
    "tz = datetime.now(timezone('Asia/Seoul'))\n",
    "time = tz.strftime('%Y-%m-%d')\n",
    "\n",
    "df['timeStamp']=time\n",
    "\n",
    "\n",
    "\n",
    "def save(collection_id, document_id, data):\n",
    "    db.collection(collection_id).document(str(document_id)).set(data)\n",
    "\n",
    "\n",
    "for i,row in df.iterrows():\n",
    "    save(\n",
    "        collection_id=\"EVENT\", \n",
    "        document_id=i,\n",
    "        data=row.to_dict()\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.9 (tensorflow)",
   "language": "python",
   "name": "tensorflow"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.15"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}