Noida Vegetable Market Price .ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "c3e49ff9",
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "import html_to_json\n",
    "from selenium import webdriver\n",
    "from selenium.webdriver.common.by import By\n",
    "from selenium.webdriver.common.keys import Keys\n",
    "from selenium.webdriver.common.action_chains import ActionChains\n",
    "import selenium\n",
    "from selenium.webdriver.support.ui import WebDriverWait\n",
    "from selenium.webdriver.support import expected_conditions as EC\n",
    "import time\n",
    "import re\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cebf93de",
   "metadata": {},
   "source": [
    "extracting list of all vegetables "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "6b14c67f",
   "metadata": {},
   "outputs": [],
   "source": [
    "link = \"https://vegetablemarketprice.com/market/noida/today\"\n",
    "\n",
    "driver = webdriver.Firefox()\n",
    "driver.get(link)\n",
    "veges = driver.find_element(By.XPATH, '//*[@id=\"todayVegetableTableContainer\"]')\n",
    "vege = veges.get_attribute(\"innerHTML\")\n",
    "\n",
    "vege_h_list = re.findall('<td>(.*?)</td>',vege)\n",
    "final_list = []\n",
    "size = len(vege_h_list) \n",
    "for i in range (size):\n",
    "    if i%5 == 0:\n",
    "        final_list.append(vege_h_list[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "13287e31",
   "metadata": {},
   "outputs": [],
   "source": [
    "final_list = ['Onion Big', 'Onion Small', 'Tomato', 'Green Chilli', 'Beetroot', 'Potato',\n",
    "    'Raw Banana (Plantain)', 'Amaranth Leaves', 'Amla', 'Ash gourd', 'Baby Corn',\n",
    "    'Banana Flower', 'Capsicum', 'Bitter Gourd', 'Bottle Gourd', 'Butter Beans',\n",
    "    'Broad Beans', 'Cabbage', 'Carrot', 'Cauliflower', 'Cluster beans', 'Coconut',\n",
    "    'Colocasia Leaves', 'Colocasia', 'Coriander Leaves', 'Corn', 'Cucumber',\n",
    "    'Curry Leaves', 'Dill Leaves', 'Drumsticks', 'Brinjal', 'Brinjal (Big)',\n",
    "    'Elephant Yam', 'Fenugreek Leaves', 'French Beans', 'Garlic', 'Ginger',\n",
    "    'Onion Green', 'Green Peas', 'Ivy Gourd', 'Lemon (Lime)', 'Mango Raw',\n",
    "    'Mint Leaves', 'Mushroom', 'Mustard Leaves', 'Ladies Finger', 'Pumpkin',\n",
    "    'Radish', 'Ridge Gourd', 'Shallot (Pearl Onion)', 'Snake Gourd', 'Sorrel Leaves',\n",
    "    'Spinach', 'Sweet Potato']\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "055ddfa1",
   "metadata": {},
   "source": [
    "Extracting dates and prices of vegetables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "304cf22b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Onion Big\n",
      "Onion Small\n",
      "Tomato\n",
      "Green Chilli\n",
      "Beetroot\n",
      "Potato\n",
      "Raw Banana (Plantain)\n",
      "Amaranth Leaves\n",
      "Amla\n",
      "Ash gourd\n",
      "Baby Corn\n",
      "Banana Flower\n"
     ]
    }
   ],
   "source": [
    "# opening a firefox browser and visiting to site \n",
    "homepage = 'https://vegetablemarketprice.com/'\n",
    "data = []\n",
    "html_list = []\n",
    "driver = webdriver.Firefox()\n",
    "driver.get(homepage)\n",
    "search_input = driver.find_element(by=By.XPATH, value='//*[@id=\"header-search-input\"]')\n",
    "\n",
    "#searching for Noida Market Price History \n",
    "header_search_input = 'Noida Vegetable Market Price History'\n",
    "search_input.send_keys(header_search_input)\n",
    "search_input.send_keys(Keys.ENTER)\n",
    "time.sleep(5)\n",
    "driver.find_element(by=By.XPATH,value='/html/body/ul/li[1]/div').click()\n",
    "\n",
    "# # switching from charts to table in dashboard\n",
    "switch = driver.find_element(By.XPATH, '//*[@id=\"navItemTable\"]')\n",
    "switch.click()\n",
    "\n",
    "#customizing the date and selecting custom date a\n",
    "change_date = driver.find_element(by = By.XPATH,value = '//*[@id=\"reportrange\"]')\n",
    "\n",
    "change_date.click()\n",
    "custom_range = driver.find_element(By.XPATH, \"//*[contains(text(), 'Custom Range')]\")\n",
    "custom_range.click()\n",
    "\n",
    "# initializing the start date\n",
    "while True: \n",
    "    check_month = driver.find_element(By.XPATH, \"/html/body/div[4]/div[2]/div[1]/table/thead/tr[1]/th[2]\")\n",
    "    month = check_month.get_attribute(\"innerHTML\")\n",
    "    if month == \"Feb 2022\":\n",
    "        break\n",
    "        \n",
    "    else:\n",
    "        back_button = driver.find_element(By.XPATH,'/html/body/div[4]/div[2]/div[1]/table/thead/tr[1]/th[1]')\n",
    "\n",
    "        back_button.click()\n",
    "        \n",
    "first_date = driver.find_element(By.XPATH, '/html/body/div[4]/div[2]/div[1]/table/tbody/tr[1]/td[3]')\n",
    "first_date.click()\n",
    "\n",
    "# switching to the next page of calender \n",
    "i = 0\n",
    "while i <= 3:\n",
    "    front_button = driver.find_element(By.XPATH,\"/html/body/div[4]/div[3]/div[1]/table/thead/tr[1]/th[3]\")\n",
    "    front_button.click()\n",
    "    i = i+1\n",
    "\n",
    "last_date = driver.find_element(By.XPATH,\"/html/body/div[4]/div[3]/div[1]/table/tbody/tr[5]/td[7]\")\n",
    "last_date.click()\n",
    "\n",
    "#applying the custom date ( feb - july)\n",
    "apply_button = driver.find_element(By.XPATH,\"/html/body/div[4]/div[4]/button[2]\")\n",
    "apply_button.click()\n",
    "\n",
    "# #Extracting  the names of vegetables available \n",
    "for i in range (0,len(final_list)):\n",
    "    while True:\n",
    "        try:\n",
    "            element = driver.find_element(By.XPATH, '/html/body/div[2]/div[2]/div/div[1]/div[4]/div/div[1]/div/div[1]/button') #removing the default names from search inputs \n",
    "            element.click()\n",
    "        except:\n",
    "            break\n",
    "# searching the prices of each vegetable from custom dates\n",
    "    search_input = driver.find_element(By.NAME, 'search_terms') \n",
    "    search_input.send_keys(final_list[i])\n",
    "    \n",
    "    search_input.send_keys(Keys.ENTER)\n",
    "    \n",
    "    temp = driver.find_element(By.XPATH, '//*[@id=\"historyDataTable\"]')\n",
    "    \n",
    "    temp_html = temp.get_attribute(\"innerHTML\")\n",
    "    html_list.append(temp_html)\n",
    "    time.sleep(10)\n",
    "\n",
    "    print(final_list[i])\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1062fcea",
   "metadata": {},
   "outputs": [],
   "source": [
    "date = re.findall('<th>(.*?)</th>',html_list[0])\n",
    "dates = []\n",
    "for i in date:\n",
    "    temp = re.findall('\"white-space: nowrap;\">(.*?)</span>',i)\n",
    "    if len(temp) != 0:\n",
    "        dates.append(temp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e487388f",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(dates,columns = ['Dates'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "05a91e1f",
   "metadata": {},
   "outputs": [],
   "source": [
    "Name = []\n",
    "Prices = []\n",
    "\n",
    "for i in html_list:\n",
    "    temp_n = re.findall('<tbody><tr><td>(.*?)</td><td>',i)\n",
    "    Name.append(temp_n)\n",
    "    \n",
    "    temp_p = re.findall('<td>₹ (.*?)</td>',i)\n",
    "    Prices.append(temp_p)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b9928f08",
   "metadata": {},
   "outputs": [],
   "source": [
    "#converting into dataframe \n",
    "df1 = pd.DataFrame(Prices)\n",
    "df1 = df1.transpose()\n",
    "df1.dropna(inplace = True)\n",
    "df1.columns = [Name]\n",
    "df = pd.merge(df,df1,left_index=True, right_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "65d3879a",
   "metadata": {},
   "outputs": [],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2472a0d6",
   "metadata": {},
   "source": [
    "## Extracting data of noida from aug to dec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "062a2bc5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# visiting to noida market price history from main page\n",
    "homepage = 'https://vegetablemarketprice.com/'\n",
    "data = []\n",
    "html_list = []\n",
    "driver = webdriver.Chrome()\n",
    "driver.get(homepage)\n",
    "search_input = driver.find_element(by=By.XPATH, value='//*[@id=\"header-search-input\"]')\n",
    "\n",
    "header_search_input = 'Noida Vegetable Market Price History'\n",
    "search_input.send_keys(header_search_input)\n",
    "search_input.send_keys(Keys.ENTER)\n",
    "time.sleep(5)\n",
    "driver.find_element(by=By.XPATH,value='/html/body/ul/li[1]/div').click()\n",
    "\n",
    "#selecting table in dashboard\n",
    "switch = driver.find_element(By.XPATH, '//*[@id=\"navItemTable\"]')\n",
    "switch.click()\n",
    "\n",
    "#selecting custom range from the date \n",
    "change_date = driver.find_element(by = By.XPATH,value = '//*[@id=\"reportrange\"]')\n",
    "change_date.click()\n",
    "custom_range = driver.find_element(By.XPATH, \"//*[contains(text(), 'Custom Range')]\")\n",
    "custom_range.click()\n",
    "\n",
    "\n",
    "# Initialzing the dates\n",
    "while True: \n",
    "    check_month = driver.find_element(By.XPATH, \"/html/body/div[4]/div[2]/div[1]/table/thead/tr[1]/th[2]\")\n",
    "    month = check_month.get_attribute(\"innerHTML\")\n",
    "    if month == \"Aug 2022\":\n",
    "        break\n",
    "        \n",
    "    else:\n",
    "        back_button = driver.find_element(By.XPATH,'/html/body/div[4]/div[2]/div[1]/table/thead/tr[1]/th[1]')\n",
    "\n",
    "        back_button.click()\n",
    "        \n",
    "first_date = driver.find_element(By.XPATH, '/html/body/div[4]/div[3]/div[1]/table/tbody/tr[1]/td[2]')\n",
    "first_date.click()\n",
    "\n",
    "#selecting date from Aug to Dec\n",
    "i=0\n",
    "while i <= 2:\n",
    "    front_button = driver.find_element(By.XPATH,\"/html/body/div[4]/div[3]/div[1]/table/thead/tr[1]/th[3]\")\n",
    "    front_button.click()\n",
    "    i = i+1\n",
    "\n",
    "last_date = driver.find_element(By.XPATH,\"/html/body/div[4]/div[3]/div[1]/table/tbody/tr[5]/td[7]\")\n",
    "last_date.click()\n",
    "\n",
    "apply_button = driver.find_element(By.XPATH,\"/html/body/div[4]/div[4]/button[2]\")\n",
    "apply_button.click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1cbebc1b",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Extracting  the names of vegetables \n",
    "for i in range (0,len(final_list)):\n",
    "    while True:\n",
    "        try:\n",
    "            element = driver.find_element(By.XPATH, '/html/body/div[2]/div[2]/div/div[1]/div[4]/div/div[1]/div/div[1]/button') # removing deafult vegeatbles presented in search \n",
    "            element.click()\n",
    "        except:\n",
    "            break\n",
    "    search_input = driver.find_element(By.NAME, 'search_terms')\n",
    "    search_input.send_keys(final_list[i]) \n",
    "    time.sleep(3)\n",
    "    search_input.send_keys(Keys.ENTER) #vegetable gets selected \n",
    "    \n",
    "    temp = driver.find_element(By.XPATH, '//*[@id=\"historyDataTable\"]') # extarcting prices \n",
    "    time.sleep(6)\n",
    "    temp_html = temp.get_attribute(\"innerHTML\")\n",
    "    html_list.append(temp_html)\n",
    "    time.sleep(6)\n",
    "\n",
    "#extracting names of vegetables and prices from html code\n",
    "Name = []\n",
    "Prices = []\n",
    "\n",
    "for i in html_list:\n",
    "    temp_n = re.findall('<tbody><tr><td>(.*?)</td><td>',i)\n",
    "    Name.extend(temp_n)\n",
    "    temp_p = re.findall('<td>₹ (.*?)</td>',i)\n",
    "    Prices.append(temp_p)\n",
    "\n",
    "#extracting date \n",
    "date = re.findall('<th>(.*?)</th>',html_list[0])\n",
    "dates = []\n",
    "for i in date:\n",
    "    temp = re.findall('\"white-space: nowrap;\">(.*?)</span>',i)\n",
    "    if len(temp) != 0:\n",
    "        dates.append(temp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a4b99147",
   "metadata": {},
   "outputs": [],
   "source": [
    "# converting into Dataframe\n",
    "df = pd.DataFrame(dates,columns = ['Dates'])\n",
    "df1 = pd.DataFrame(Prices).transpose()\n",
    "df1.dropna(inplace = True)\n",
    "df1.columns = [Name]\n",
    "df = pd.merge(df,df1,left_index=True, right_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "551f9979",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Dates</th>\n",
       "      <th>(Onion Small,)</th>\n",
       "      <th>(Onion Green,)</th>\n",
       "      <th>(Sweet Potato,)</th>\n",
       "      <th>(French Beans,)</th>\n",
       "      <th>(Sweet Potato,)</th>\n",
       "      <th>(Potato,)</th>\n",
       "      <th>(Raw Banana (Plantain),)</th>\n",
       "      <th>(Mustard Leaves,)</th>\n",
       "      <th>(Cabbage,)</th>\n",
       "      <th>...</th>\n",
       "      <th>(Mint Leaves,)</th>\n",
       "      <th>(Mint Leaves,)</th>\n",
       "      <th>(Mint Leaves,)</th>\n",
       "      <th>(Mint Leaves,)</th>\n",
       "      <th>(Mint Leaves,)</th>\n",
       "      <th>(Shallot (Pearl Onion),)</th>\n",
       "      <th>(Ridge Gourd,)</th>\n",
       "      <th>(Mustard Leaves,)</th>\n",
       "      <th>(Snake Gourd,)</th>\n",
       "      <th>(Beetroot,)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2022-07-31</td>\n",
       "      <td>32</td>\n",
       "      <td>42</td>\n",
       "      <td>36</td>\n",
       "      <td>64</td>\n",
       "      <td>36</td>\n",
       "      <td>33</td>\n",
       "      <td>9</td>\n",
       "      <td>12</td>\n",
       "      <td>21</td>\n",
       "      <td>...</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>37</td>\n",
       "      <td>25</td>\n",
       "      <td>12</td>\n",
       "      <td>28</td>\n",
       "      <td>41</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2022-08-01</td>\n",
       "      <td>30</td>\n",
       "      <td>42</td>\n",
       "      <td>40</td>\n",
       "      <td>67</td>\n",
       "      <td>40</td>\n",
       "      <td>30</td>\n",
       "      <td>8</td>\n",
       "      <td>12</td>\n",
       "      <td>21</td>\n",
       "      <td>...</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>32</td>\n",
       "      <td>29</td>\n",
       "      <td>12</td>\n",
       "      <td>27</td>\n",
       "      <td>40</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2022-08-02</td>\n",
       "      <td>30</td>\n",
       "      <td>42</td>\n",
       "      <td>40</td>\n",
       "      <td>67</td>\n",
       "      <td>40</td>\n",
       "      <td>30</td>\n",
       "      <td>8</td>\n",
       "      <td>12</td>\n",
       "      <td>21</td>\n",
       "      <td>...</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>32</td>\n",
       "      <td>29</td>\n",
       "      <td>12</td>\n",
       "      <td>27</td>\n",
       "      <td>40</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2022-08-03</td>\n",
       "      <td>30</td>\n",
       "      <td>42</td>\n",
       "      <td>40</td>\n",
       "      <td>67</td>\n",
       "      <td>40</td>\n",
       "      <td>30</td>\n",
       "      <td>8</td>\n",
       "      <td>12</td>\n",
       "      <td>21</td>\n",
       "      <td>...</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>32</td>\n",
       "      <td>29</td>\n",
       "      <td>12</td>\n",
       "      <td>27</td>\n",
       "      <td>40</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2022-08-04</td>\n",
       "      <td>31</td>\n",
       "      <td>41</td>\n",
       "      <td>40</td>\n",
       "      <td>63</td>\n",
       "      <td>40</td>\n",
       "      <td>31</td>\n",
       "      <td>10</td>\n",
       "      <td>16</td>\n",
       "      <td>21</td>\n",
       "      <td>...</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>33</td>\n",
       "      <td>32</td>\n",
       "      <td>16</td>\n",
       "      <td>26</td>\n",
       "      <td>43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>149</th>\n",
       "      <td>2022-12-27</td>\n",
       "      <td>38</td>\n",
       "      <td>42</td>\n",
       "      <td>37</td>\n",
       "      <td>27</td>\n",
       "      <td>37</td>\n",
       "      <td>27</td>\n",
       "      <td>9</td>\n",
       "      <td>18</td>\n",
       "      <td>30</td>\n",
       "      <td>...</td>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "      <td>36</td>\n",
       "      <td>25</td>\n",
       "      <td>18</td>\n",
       "      <td>28</td>\n",
       "      <td>39</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>150</th>\n",
       "      <td>2022-12-28</td>\n",
       "      <td>38</td>\n",
       "      <td>42</td>\n",
       "      <td>37</td>\n",
       "      <td>27</td>\n",
       "      <td>37</td>\n",
       "      <td>27</td>\n",
       "      <td>9</td>\n",
       "      <td>18</td>\n",
       "      <td>30</td>\n",
       "      <td>...</td>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "      <td>36</td>\n",
       "      <td>25</td>\n",
       "      <td>18</td>\n",
       "      <td>28</td>\n",
       "      <td>39</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>151</th>\n",
       "      <td>2022-12-29</td>\n",
       "      <td>39</td>\n",
       "      <td>41</td>\n",
       "      <td>38</td>\n",
       "      <td>30</td>\n",
       "      <td>38</td>\n",
       "      <td>32</td>\n",
       "      <td>9</td>\n",
       "      <td>18</td>\n",
       "      <td>24</td>\n",
       "      <td>...</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>36</td>\n",
       "      <td>28</td>\n",
       "      <td>18</td>\n",
       "      <td>28</td>\n",
       "      <td>37</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>152</th>\n",
       "      <td>2022-12-30</td>\n",
       "      <td>39</td>\n",
       "      <td>41</td>\n",
       "      <td>38</td>\n",
       "      <td>30</td>\n",
       "      <td>38</td>\n",
       "      <td>32</td>\n",
       "      <td>9</td>\n",
       "      <td>18</td>\n",
       "      <td>24</td>\n",
       "      <td>...</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>36</td>\n",
       "      <td>28</td>\n",
       "      <td>18</td>\n",
       "      <td>28</td>\n",
       "      <td>37</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>153</th>\n",
       "      <td>2022-12-31</td>\n",
       "      <td>39</td>\n",
       "      <td>41</td>\n",
       "      <td>38</td>\n",
       "      <td>30</td>\n",
       "      <td>38</td>\n",
       "      <td>32</td>\n",
       "      <td>9</td>\n",
       "      <td>18</td>\n",
       "      <td>24</td>\n",
       "      <td>...</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>36</td>\n",
       "      <td>28</td>\n",
       "      <td>18</td>\n",
       "      <td>28</td>\n",
       "      <td>37</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>154 rows × 55 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          Dates (Onion Small,) (Onion Green,) (Sweet Potato,) (French Beans,)  \\\n",
       "0    2022-07-31             32             42              36              64   \n",
       "1    2022-08-01             30             42              40              67   \n",
       "2    2022-08-02             30             42              40              67   \n",
       "3    2022-08-03             30             42              40              67   \n",
       "4    2022-08-04             31             41              40              63   \n",
       "..          ...            ...            ...             ...             ...   \n",
       "149  2022-12-27             38             42              37              27   \n",
       "150  2022-12-28             38             42              37              27   \n",
       "151  2022-12-29             39             41              38              30   \n",
       "152  2022-12-30             39             41              38              30   \n",
       "153  2022-12-31             39             41              38              30   \n",
       "\n",
       "    (Sweet Potato,) (Potato,) (Raw Banana (Plantain),) (Mustard Leaves,)  \\\n",
       "0                36        33                        9                12   \n",
       "1                40        30                        8                12   \n",
       "2                40        30                        8                12   \n",
       "3                40        30                        8                12   \n",
       "4                40        31                       10                16   \n",
       "..              ...       ...                      ...               ...   \n",
       "149              37        27                        9                18   \n",
       "150              37        27                        9                18   \n",
       "151              38        32                        9                18   \n",
       "152              38        32                        9                18   \n",
       "153              38        32                        9                18   \n",
       "\n",
       "    (Cabbage,)  ... (Mint Leaves,) (Mint Leaves,) (Mint Leaves,)  \\\n",
       "0           21  ...              4              4              4   \n",
       "1           21  ...              5              5              5   \n",
       "2           21  ...              5              5              5   \n",
       "3           21  ...              5              5              5   \n",
       "4           21  ...              5              5              5   \n",
       "..         ...  ...            ...            ...            ...   \n",
       "149         30  ...              6              6              6   \n",
       "150         30  ...              6              6              6   \n",
       "151         24  ...              4              4              4   \n",
       "152         24  ...              4              4              4   \n",
       "153         24  ...              4              4              4   \n",
       "\n",
       "    (Mint Leaves,) (Mint Leaves,) (Shallot (Pearl Onion),) (Ridge Gourd,)  \\\n",
       "0                4              4                       37             25   \n",
       "1                5              5                       32             29   \n",
       "2                5              5                       32             29   \n",
       "3                5              5                       32             29   \n",
       "4                5              5                       33             32   \n",
       "..             ...            ...                      ...            ...   \n",
       "149              6              6                       36             25   \n",
       "150              6              6                       36             25   \n",
       "151              4              4                       36             28   \n",
       "152              4              4                       36             28   \n",
       "153              4              4                       36             28   \n",
       "\n",
       "    (Mustard Leaves,) (Snake Gourd,) (Beetroot,)  \n",
       "0                  12             28          41  \n",
       "1                  12             27          40  \n",
       "2                  12             27          40  \n",
       "3                  12             27          40  \n",
       "4                  16             26          43  \n",
       "..                ...            ...         ...  \n",
       "149                18             28          39  \n",
       "150                18             28          39  \n",
       "151                18             28          37  \n",
       "152                18             28          37  \n",
       "153                18             28          37  \n",
       "\n",
       "[154 rows x 55 columns]"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6d9dc5b1",
   "metadata": {},
   "outputs": [],
   "source": [
    "# exporting the data into csv file\n",
    "path = 'C:\\\\Users\\\\91982\\\\Downloads\\\\project\\\\Feb_july_Prices_Noida.csv'\n",
    "df.to_csv(path, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c0727a27",
   "metadata": {},
   "outputs": [],
   "source": [
    "new_list=[]\n",
    "for i in final_list:\n",
    "    if i not in Name:\n",
    "        new_list.append(i)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "953e40cc",
   "metadata": {},
   "source": [
    "## Extracting data of noida from Jan to March "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1804d986",
   "metadata": {},
   "outputs": [],
   "source": [
    "#visiting the page\n",
    "homepage = 'https://vegetablemarketprice.com/'\n",
    "data = []\n",
    "html_list = []\n",
    "driver = webdriver.Firefox()\n",
    "driver.get(homepage)\n",
    "search_input = driver.find_element(by=By.XPATH, value='//*[@id=\"header-search-input\"]')\n",
    "\n",
    "#Searching Noida Price History\n",
    "header_search_input = 'Noida Vegetable Market Price History'\n",
    "search_input.send_keys(header_search_input)\n",
    "search_input.send_keys(Keys.ENTER)\n",
    "time.sleep(5)\n",
    "driver.find_element(by=By.XPATH,value='/html/body/ul/li[1]/div').click()\n",
    "\n",
    "#switching from chart to table in dashboard\n",
    "switch = driver.find_element(By.XPATH, '//*[@id=\"navItemTable\"]')\n",
    "switch.click()\n",
    "\n",
    "#selecting the date \n",
    "change_date = driver.find_element(by = By.XPATH,value = '//*[@id=\"reportrange\"]')\n",
    "change_date.click()\n",
    "custom_range = driver.find_element(By.XPATH, \"//*[contains(text(), 'Custom Range')]\")\n",
    "custom_range.click()\n",
    "\n",
    "# Initialzing starting date and end date\n",
    "while True: \n",
    "    check_month = driver.find_element(By.XPATH, \"/html/body/div[4]/div[2]/div[1]/table/thead/tr[1]/th[2]\")\n",
    "    month = check_month.get_attribute(\"innerHTML\")\n",
    "    if month == \"Jan 2023\":\n",
    "        break\n",
    "        \n",
    "    else:\n",
    "        back_button = driver.find_element(By.XPATH,'/html/body/div[4]/div[2]/div[1]/table/thead/tr[1]/th[1]')\n",
    "\n",
    "        back_button.click()\n",
    "        \n",
    "first_date = driver.find_element(By.XPATH, '/html/body/div[4]/div[2]/div[1]/table/tbody/tr[2]/td[1]')\n",
    "first_date.click()\n",
    "\n",
    "i=0                  #switching page of calendar \n",
    "while i < 1:\n",
    "    front_button = driver.find_element(By.XPATH,\"/html/body/div[4]/div[3]/div[1]/table/thead/tr[1]/th[3]\")\n",
    "    front_button.click()\n",
    "    i = i+1\n",
    "\n",
    "last_date = driver.find_element(By.XPATH,\"/html/body/div[4]/div[3]/div[1]/table/tbody/tr[5]/td[6]\")  # selecting end date \n",
    "last_date.click()\n",
    "\n",
    "#applyig the dates \n",
    "wait = WebDriverWait(driver, 60)  # Adjust the timeout as needed\n",
    "apply_button = wait.until(EC.element_to_be_clickable((By.XPATH, \"/html/body/div[4]/div[4]/button[2]\")))\n",
    "apply_button.click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "afd77ef8",
   "metadata": {},
   "outputs": [],
   "source": [
    "# extracing the names of vegetables\n",
    "for i in range (0,len(final_list)):\n",
    "    while True:\n",
    "        try:\n",
    "            element = driver.find_element(By.XPATH, '/html/body/div[2]/div[2]/div/div[1]/div[4]/div/div[1]/div/div[1]/button')\n",
    "            element.click()\n",
    "        except:\n",
    "            break\n",
    "    search_input = driver.find_element(By.NAME, 'search_terms')\n",
    "    search_input.send_keys(final_list[i])\n",
    "    time.sleep(3)\n",
    "    search_input.send_keys(Keys.ENTER)\n",
    "    \n",
    "    temp = driver.find_element(By.XPATH, '//*[@id=\"historyDataTable\"]')\n",
    "    time.sleep(3)\n",
    "    temp_html = temp.get_attribute(\"innerHTML\")\n",
    "    html_list.append(temp_html)\n",
    "    time.sleep(3)\n",
    "    print(final_list[i])\n",
    "\n",
    "# extracting the dates from html code\n",
    "date = re.findall('<th>(.*?)</th>',html_list[0])\n",
    "dates = []\n",
    "for i in date:\n",
    "    temp = re.findall('\"white-space: nowrap;\">(.*?)</span>',i)\n",
    "    if len(temp) != 0:\n",
    "        dates.append(temp)\n",
    "\n",
    "\n",
    "# Extracting names and prices of the vegetables \n",
    "Name = []\n",
    "Prices = []\n",
    "\n",
    "for i in html_list:\n",
    "    temp_n = re.findall('<tbody><tr><td>(.*?)</td><td>',i)\n",
    "    Name.extend(temp_n)\n",
    "    \n",
    "    temp_p = re.findall('<td>₹ (.*?)</td>',i)\n",
    "    Prices.append(temp_p)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0efa0857",
   "metadata": {},
   "outputs": [],
   "source": [
    "#creating a dataframe\n",
    "df = pd.DataFrame(dates,columns = ['Dates'])\n",
    "df1 = pd.DataFrame(Prices).transpose()\n",
    "df1.dropna(inplace = True)\n",
    "df1.columns = [Name]\n",
    "df = pd.merge(df,df1,left_index=True, right_index=True)\n",
    "\n",
    "# exporting csv file \n",
    "path = 'C:\\\\Users\\\\91982\\\\Downloads\\\\project\\\\AN_MARCH_Prices_Noida.csv'\n",
    "df.to_csv(path, index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}