三立新聞/SETN_crawler.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-08T05:30:13.465514Z",
     "start_time": "2021-03-08T05:30:11.983780Z"
    }
   },
   "outputs": [],
   "source": [
    "from selenium import webdriver\n",
    "from multiprocessing.dummy import Pool as ThreadPool\n",
    "from selenium.webdriver.chrome.options import Options\n",
    "import json\n",
    "import time\n",
    "from tqdm import tqdm\n",
    "import os\n",
    "import pandas as pd\n",
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "import numpy as np\n",
    "\n",
    "chrome_options = Options()\n",
    "chrome_options.add_argument('--no-sandbox')\n",
    "chrome_options.add_argument('--disable-dev-shm-usage')\n",
    "chrome_options.add_argument('--headless')\n",
    "chrome_options.add_argument('blink-settings=imagesEnabled=false')\n",
    "chrome_options.add_argument('--disable-gpu')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-27T16:29:54.198143Z",
     "start_time": "2021-02-27T16:29:54.191880Z"
    }
   },
   "outputs": [],
   "source": [
    "#初始化啟動chrome webdriver\n",
    "driverpath= \"../Driver/chromedriver88OS\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-26T06:49:32.349351Z",
     "start_time": "2021-02-26T06:49:32.322617Z"
    }
   },
   "outputs": [],
   "source": [
    "#Read Legislator List\n",
    "df = pd.read_csv('./data/demo_data/立委名單.txt', sep='/n', encoding='utf-8')\n",
    "legislator = list(set(list(df['立委姓名'])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-26T06:49:39.477089Z",
     "start_time": "2021-02-26T06:49:32.820447Z"
    }
   },
   "outputs": [],
   "source": [
    "browser=webdriver.Chrome(executable_path=driverpath)\n",
    "browser.implicitly_wait(30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-26T06:49:41.520920Z",
     "start_time": "2021-02-26T06:49:39.480090Z"
    }
   },
   "outputs": [],
   "source": [
    "link = {}\n",
    "for i,name in enumerate(legislator):\n",
    "    l=[]\n",
    "    for page in range(500):\n",
    "        url=\"https://www.setn.com/search.aspx?q={}&r=0&p={}\".format(name, page)\n",
    "        browser.get(url)\n",
    "        css = browser.find_element_by_css_selector(\"div.col-md-9.col-sm-12.contLeft\")\n",
    "        gt = css.find_elements_by_class_name('gt')\n",
    "        l_temp=[]\n",
    "        for element in gt:\n",
    "            l_temp.append(element.get_attribute('href'))\n",
    "        l_temp = l_temp[0::2]\n",
    "        if len(l_temp)!=36:\n",
    "            link[name]=list(set(l))\n",
    "            print(len(link[name]))\n",
    "            break\n",
    "        else:\n",
    "            l.extend(l_temp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-25T18:22:31.501127Z",
     "start_time": "2021-02-25T18:22:31.404028Z"
    }
   },
   "outputs": [],
   "source": [
    "with open('./data/link.json', 'w') as f:\n",
    "    json.dump(link, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-08T05:30:19.798163Z",
     "start_time": "2021-03-08T05:30:19.777085Z"
    }
   },
   "outputs": [],
   "source": [
    "with open('./data/link.json', 'r') as f:\n",
    "    link = json.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-01T18:06:29.316212Z",
     "start_time": "2021-03-01T18:06:29.243230Z"
    }
   },
   "outputs": [],
   "source": [
    "def crawler_article(data):\n",
    "    name = data[0]\n",
    "    urls = data[1]\n",
    "    \n",
    "    if (urls!=[]) & (name  not in [i.split('.')[0] for i in os.listdir('./data')]):\n",
    "        \n",
    "        print(\"Start crawler news of {}\".format(name))\n",
    "\n",
    "        Title = []\n",
    "        Time = []\n",
    "        Category = []\n",
    "        Hashtag = []\n",
    "        Content = []\n",
    "        Link = []\n",
    "        Error_list = []\n",
    "        \n",
    "        print('Number of news of {}:'.format(name), len(urls))\n",
    "        for url in urls:\n",
    "            #print(\"Browser Get Success\")\n",
    "            resp = requests.get(url)\n",
    "            if resp.status_code != 200:\n",
    "                Error_list.append(url)\n",
    "            else:\n",
    "                try:\n",
    "                    soup = BeautifulSoup(resp.text, 'lxml')\n",
    "\n",
    "                    #Crawler Title\n",
    "                    #print('Crawler Title')\n",
    "                    title = soup.find('h1', 'news-title-3').string.strip()\n",
    "\n",
    "                    #Crawler Time\n",
    "                    #print('Crawler Time')\n",
    "                    time = soup.find('time', 'page-date').string.strip()\n",
    "\n",
    "                    #Crawler Category\n",
    "                    #print('Crawler Category')\n",
    "                    category = soup.find('meta', property=\"article:section\").get('content')\n",
    "\n",
    "                    #Crawler Hashtag\n",
    "                    #print(\"Crawler Hashtag\")\n",
    "                    hashtag = soup.find('meta', itemprop=\"keywords\").get('content')\n",
    "                    hashtag = \"#\"+\"#\".join(hashtag.split(','))\n",
    "\n",
    "                    #Crawler Content\n",
    "                    #print(\"Crawler Content\")\n",
    "                    C1 = soup.find_all('p')\n",
    "                    content=''\n",
    "                    for p in C1:\n",
    "                        if \"▲\" in p.text or p.text == '':\n",
    "                            continue\n",
    "                        else:\n",
    "                            content+=p.text.strip()+'\\n'\n",
    "                except:\n",
    "                    print(url)\n",
    "                        \n",
    "                if content.replace('\\n', '') == '':\n",
    "                    continue\n",
    "\n",
    "                Title.append(title)\n",
    "                Time.append(time)\n",
    "                Category.append(category)\n",
    "                Hashtag.append(hashtag)\n",
    "                Content.append(content)\n",
    "                Link.append(url)\n",
    "\n",
    "        print(\"{} Crawler Done!\".format(name))\n",
    "\n",
    "        df = pd.DataFrame({\"TITLE\":Title, \"政治野心\": np.nan, \"TIME\":Time, \"CATEGORY\":Category, \"HASHTAG\":Hashtag, \"Content\":Content, \"SOURCE\":['三立新聞網']*len(Title), \"LINK\":Link})\n",
    "\n",
    "        print('Save {} as XLSX'.format(name))\n",
    "        df.to_excel(\"./data/{}.SETN.xlsx\".format(name), encoding='utf-8-sig', index=False)\n",
    "        return name, Error_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-01T18:06:30.068273Z",
     "start_time": "2021-03-01T18:06:30.065254Z"
    }
   },
   "outputs": [],
   "source": [
    "error_website={}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-01T18:06:30.388929Z",
     "start_time": "2021-03-01T18:06:30.384776Z"
    }
   },
   "outputs": [],
   "source": [
    "run = [(name, link[name]) for name in link]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-01T18:06:31.008347Z",
     "start_time": "2021-03-01T18:06:30.790576Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "pool = ThreadPool(4)\n",
    "try:\n",
    "    name, Error_list = pool.map(crawler_article, run)\n",
    "    error_website[name] = Error_list\n",
    "except:\n",
    "    pass\n",
    "pool.close()\n",
    "pool.join()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}