民生公共物聯網.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "歷史資料-環保署國家空品測站監測資料\n",
      "https://ci.taiwan.gov.tw/dsp/history/epa_erdb/\n",
      "歷史資料-環保署智慧城鄉空品微型感測器監測資料\n",
      "https://ci.taiwan.gov.tw/dsp/history/epa_micro/\n",
      "歷史資料-中研院校園空品微型感測器監測資料\n",
      "https://ci.taiwan.gov.tw/dsp/history/iis_airbox/\n",
      "歷史資料-中央氣象局地震監測站\n",
      "https://ci.taiwan.gov.tw/dsp/history/eq_cwb/\n",
      "歷史資料-國家地震工程研究中心地震監測站\n",
      "https://ci.taiwan.gov.tw/dsp/history/eq_ncree/\n",
      "歷史資料-災害示警歷史資料\n",
      "https://ci.taiwan.gov.tw/dsp/history/cap/\n",
      "歷史資料-災情通報資料\n",
      "https://ci.taiwan.gov.tw/dsp/history/nfa/\n",
      "歷史資料-中央氣象局局屬氣象站（現在天氣觀測報告）\n",
      "https://ci.taiwan.gov.tw/dsp/history/cwb_metro/\n",
      "歷史資料-中央氣象局自動氣象站（氣象觀測資料）\n",
      "https://ci.taiwan.gov.tw/dsp/history/cwb_auto/\n",
      "歷史資料-中央氣象局自動雨量站（雨量觀測資料）\n",
      "https://ci.taiwan.gov.tw/dsp/history/cwb_rain/\n",
      "歷史資料-科技部智慧園區空品測站監測資料\n",
      "https://ci.taiwan.gov.tw/dsp/history/most_aq/\n",
      "歷史資料-大同空品微型感測器監測資料\n",
      "https://ci.taiwan.gov.tw/dsp/history/aq_tatung/\n",
      "歷史資料-在地空品微型感測器資料\n",
      "https://ci.taiwan.gov.tw/dsp/history/aq_local/\n",
      "歷史資料-中央氣象局地磁觀測站\n",
      "https://ci.taiwan.gov.tw/dsp/history/eq_cwb_mag/\n",
      "歷史資料-中央氣象局地震地下水觀測站\n",
      "https://ci.taiwan.gov.tw/dsp/history/eq_cwb_gw/\n",
      "歷史資料-水利署_河川水位站\n",
      "https://ci.taiwan.gov.tw/dsp/history/wra_iow_水利署_河川水位站/\n",
      "歷史資料-水利署_雨量感測器\n",
      "https://ci.taiwan.gov.tw/dsp/history/wra_iow_水利署_雨量感測器/\n",
      "歷史資料-水利署_淹水感測器\n",
      "https://ci.taiwan.gov.tw/dsp/history/wra_iow_水利署_淹水感測器/\n",
      "歷史資料-水利署_閘門\n",
      "https://ci.taiwan.gov.tw/dsp/history/wra_iow_水利署_閘門/\n",
      "歷史資料-水利署（與縣市政府合建）_雨量感測器\n",
      "https://ci.taiwan.gov.tw/dsp/history/wra_iow_水利署（與縣市政府合建）_雨量感測器/\n",
      "歷史資料-水利署（與縣市政府合建）_區域排水水位站\n",
      "https://ci.taiwan.gov.tw/dsp/history/wra_iow_水利署（與縣市政府合建）_區域排水水位站/\n",
      "歷史資料-水利署（與縣市政府合建）_淹水感測器\n",
      "https://ci.taiwan.gov.tw/dsp/history/wra_iow_水利署（與縣市政府合建）_淹水感測器/\n",
      "歷史資料-水利署（與縣市政府合建）_閘門\n",
      "https://ci.taiwan.gov.tw/dsp/history/wra_iow_水利署（與縣市政府合建）_閘門/\n",
      "歷史資料-農田水利署_雨量感測器\n",
      "https://ci.taiwan.gov.tw/dsp/history/wra_iow_農田水利署_雨量感測器/\n",
      "歷史資料-農田水利署_流量感測器\n",
      "https://ci.taiwan.gov.tw/dsp/history/wra_iow_農田水利署_流量感測器/\n",
      "歷史資料-農田水利署_埤塘水位站\n",
      "https://ci.taiwan.gov.tw/dsp/history/wra_iow_農田水利署_埤塘水位站/\n",
      "歷史資料-農田水利署_農田灌溉圳路水位站\n",
      "https://ci.taiwan.gov.tw/dsp/history/wra_iow_農田水利署_農田灌溉圳路水位站/\n",
      "歷史資料-農田水利署_閘門\n",
      "https://ci.taiwan.gov.tw/dsp/history/wra_iow_農田水利署_閘門/\n",
      "歷史資料-營建署_化學需氧量感測器\n",
      "https://ci.taiwan.gov.tw/dsp/history/wra_iow_營建署_化學需氧量感測器/\n",
      "歷史資料-營建署_污水放水量感測器\n",
      "https://ci.taiwan.gov.tw/dsp/history/wra_iow_營建署_污水放水量感測器/\n",
      "歷史資料-營建署_懸浮固體感測器\n",
      "https://ci.taiwan.gov.tw/dsp/history/wra_iow_營建署_懸浮固體感測器/\n",
      "歷史資料-營建署_雨水下水道水位計\n",
      "https://ci.taiwan.gov.tw/dsp/history/groundwater_capmi/\n",
      "歷史資料-水利署_堤防結構安全測站\n",
      "https://ci.taiwan.gov.tw/dsp/history/wra_iow_水利署_堤防結構安全測站/\n",
      "歷史資料-水利署（與縣市政府合建）_流量感測器\n",
      "https://ci.taiwan.gov.tw/dsp/history/wra_iow_水利署（與縣市政府合建）_流量感測器/\n",
      "歷史資料-水利署（與縣市政府合建）_移動抽水機\n",
      "https://ci.taiwan.gov.tw/dsp/history/wra_iow_水利署（與縣市政府合建）_移動抽水機/\n",
      "歷史資料-臺北市_抽水站\n",
      "https://ci.taiwan.gov.tw/dsp/history/pump_taipei/\n",
      "歷史資料-水利署_非連續性淹水感測器\n",
      "https://ci.taiwan.gov.tw/dsp/history/wra_iow_水利署_非連續性淹水感測器/\n",
      "歷史資料-水利署（與縣市政府合建）_非連續性淹水感測器\n",
      "https://ci.taiwan.gov.tw/dsp/history/wra_iow_水利署（與縣市政府合建）_非連續性淹水感測器/\n",
      "歷史資料-水利署（與縣市政府合建）_抽水站\n",
      "https://ci.taiwan.gov.tw/dsp/history/wra_iow_水利署（與縣市政府合建）_抽水站/\n",
      "歷史資料-中央氣象局全球導航衛星系統觀測站\n",
      "https://ci.taiwan.gov.tw/dsp/history/eq_cwb_gnss/\n"
     ]
    }
   ],
   "source": [
    "import requests\n",
    "import os\n",
    "from bs4 import BeautifulSoup\n",
    "\n",
    "url = 'https://ci.taiwan.gov.tw/dsp/environmental.aspx'\n",
    "\n",
    "re = requests.get(url)\n",
    "soup = BeautifulSoup(re.text,'html.parser')\n",
    "\n",
    "c = soup.findAll('div', {'class':'modal-body'})\n",
    "\n",
    "item_dict={}\n",
    "for i in c:\n",
    "    if i.find('iframe') == None:\n",
    "        continue\n",
    "    else:\n",
    "        title = i.find('iframe')['title']\n",
    "        Link = i.find('iframe')['src']\n",
    "        item_dict[title] = Link\n",
    "        print(title)\n",
    "        print(Link)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def folder_download(option,url):\n",
    "    for o in option:\n",
    "        new_url = url +'/?dir='+ o\n",
    "\n",
    "        re = requests.get(new_url)\n",
    "        soup = BeautifulSoup(re.text,'html.parser')\n",
    "        tag_a = soup.findAll('a')\n",
    "\n",
    "        for i in tag_a:\n",
    "            if 'zip' not in i['href']:\n",
    "                continue\n",
    "            else:\n",
    "                link = url + i['href']\n",
    "                file_name = '{}/{}/{}'.format(path,o,i['data-name'].strip('wra_iow_').replace('/','_'))\n",
    "                if not os.path.exists(os.path.join(path,o)):\n",
    "                    os.makedirs(os.path.join(path,o))\n",
    "                if not os.path.exists(file_name):\n",
    "                    with open(file_name, 'wb') as f:\n",
    "                        r = requests.get(link)\n",
    "                        f.write(r.content)\n",
    "                        r.close()\n",
    "\n",
    "def download(tag,url):\n",
    "    link = url + tag['data-name']\n",
    "    file_name = '{}/{}'.format(path,tag['data-name'])\n",
    "    if not os.path.exists(file_name):\n",
    "        with open(file_name, 'wb') as f:\n",
    "            r = requests.get(link)\n",
    "            f.write(r.content)\n",
    "            r.close()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['歷史資料-水利署_河川水位站', '歷史資料-水利署_雨量感測器', '歷史資料-水利署_淹水感測器', '歷史資料-水利署_閘門', '歷史資料-水利署（與縣市政府合建）_雨量感測器', '歷史資料-水利署（與縣市政府合建）_區域排水水位站', '歷史資料-水利署（與縣市政府合建）_淹水感測器', '歷史資料-水利署（與縣市政府合建）_閘門', '歷史資料-農田水利署_雨量感測器', '歷史資料-農田水利署_流量感測器', '歷史資料-農田水利署_埤塘水位站', '歷史資料-農田水利署_農田灌溉圳路水位站', '歷史資料-農田水利署_閘門', '歷史資料-營建署_化學需氧量感測器', '歷史資料-營建署_污水放水量感測器', '歷史資料-營建署_懸浮固體感測器', '歷史資料-營建署_雨水下水道水位計', '歷史資料-中央氣象局局屬氣象站（現在天氣觀測報告）', '歷史資料-中央氣象局自動氣象站（氣象觀測資料）', '歷史資料-中央氣象局自動雨量站（雨量觀測資料）']\n"
     ]
    }
   ],
   "source": [
    "target = ['歷史資料-水利署_河川水位站',                         #1\n",
    "          '歷史資料-水利署_雨量感測器',                         #2\n",
    "          '歷史資料-水利署_淹水感測器',                         #3\n",
    "          '歷史資料-水利署_閘門',                              #4\n",
    "          '歷史資料-水利署（與縣市政府合建）_雨量感測器',        #5\n",
    "          '歷史資料-水利署（與縣市政府合建）_區域排水水位站',     #6\n",
    "          '歷史資料-水利署（與縣市政府合建）_淹水感測器',        #7\n",
    "          '歷史資料-水利署（與縣市政府合建）_閘門',              #8\n",
    "          '歷史資料-農田水利署_雨量感測器',                         #9\n",
    "          '歷史資料-農田水利署_流量感測器',                         #10\n",
    "          '歷史資料-農田水利署_埤塘水位站',                         #11\n",
    "          '歷史資料-農田水利署_農田灌溉圳路水位站',                  #12\n",
    "          '歷史資料-農田水利署_閘門',                              #13\n",
    "          '歷史資料-營建署_化學需氧量感測器',                   #14\n",
    "          '歷史資料-營建署_污水放水量感測器',                   #15\n",
    "          '歷史資料-營建署_懸浮固體感測器',                     #16\n",
    "          '歷史資料-營建署_雨水下水道水位計',                   #17\n",
    "          '歷史資料-中央氣象局局屬氣象站（現在天氣觀測報告）',    #18\n",
    "          '歷史資料-中央氣象局自動氣象站（氣象觀測資料）',       #19\n",
    "          '歷史資料-中央氣象局自動雨量站（雨量觀測資料）']       #20\n",
    "\n",
    "#選擇要下載的項目\n",
    "choose = [int(m)-1 for m in input().split(' ')]\n",
    "choose_lst = [target[n] for n in choose]\n",
    "\n",
    "#全部更新\n",
    "#choose_lst = target         \n",
    "\n",
    "print(choose_lst)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "歷史資料-水利署_河川水位站 done\n",
      "歷史資料-水利署_雨量感測器 done\n",
      "歷史資料-水利署_淹水感測器 done\n",
      "歷史資料-水利署_閘門 done\n",
      "歷史資料-水利署（與縣市政府合建）_雨量感測器 done\n",
      "歷史資料-水利署（與縣市政府合建）_區域排水水位站 done\n",
      "歷史資料-水利署（與縣市政府合建）_淹水感測器 done\n",
      "歷史資料-水利署（與縣市政府合建）_閘門 done\n",
      "歷史資料-農田水利署_雨量感測器 done\n",
      "歷史資料-農田水利署_流量感測器 done\n",
      "歷史資料-農田水利署_埤塘水位站 done\n",
      "歷史資料-農田水利署_農田灌溉圳路水位站 done\n",
      "歷史資料-農田水利署_閘門 done\n",
      "歷史資料-營建署_化學需氧量感測器 done\n",
      "歷史資料-營建署_污水放水量感測器 done\n",
      "歷史資料-營建署_懸浮固體感測器 done\n",
      "歷史資料-營建署_雨水下水道水位計 done\n",
      "歷史資料-中央氣象局局屬氣象站（現在天氣觀測報告） done\n",
      "歷史資料-中央氣象局自動氣象站（氣象觀測資料） done\n",
      "歷史資料-中央氣象局自動雨量站（雨量觀測資料） done\n",
      "finished\n"
     ]
    }
   ],
   "source": [
    "import requests\n",
    "import os\n",
    "from bs4 import BeautifulSoup\n",
    "\n",
    "\n",
    "for t in choose_lst:\n",
    "    #main_path = '/Users/johnny/Desktop/download'     #存到資料夾\n",
    "    main_path = 'E:\\民生'                             #存到隨身碟\n",
    "    url = item_dict[t]\n",
    "\n",
    "    re = requests.get(url)\n",
    "    soup = BeautifulSoup(re.text,'html.parser')\n",
    "    li_tag = soup.findAll('li')\n",
    "\n",
    "    path = main_path +'/'+ t.split('-')[1]\n",
    "    if not os.path.exists(path):\n",
    "        os.makedirs(path)\n",
    "\n",
    "    option = []   \n",
    "    for j in li_tag: \n",
    "        if 'zip' not in j['data-name']:\n",
    "            if 'pdf' in j['data-name']:\n",
    "                download(j,url)\n",
    "            elif 'csv' in j['data-name']:\n",
    "                download(j,url)\n",
    "            else:\n",
    "                option.append(j['data-name'])\n",
    "        else:\n",
    "            download(j,url)\n",
    "    folder_download(option,url)\n",
    "    print(t,'done')\n",
    "\n",
    "print('finished')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}