geocode_v_2.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "5Oa-0CF5QRl-"
   },
   "source": [
    "#**Geocode version 2.0**\n",
    "\n",
    "data is taken from economic census of banglore(BBMP). \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "H9nEnMx0RT3R"
   },
   "source": [
    "***import libraries***"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "uAKpm7ckLf1S"
   },
   "outputs": [],
   "source": [
    "import requests\n",
    "import time\n",
    "import json\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "0yBfbSogReb4"
   },
   "source": [
    "places.csv is extracted details of places , streets and addresses from BBMP data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "wU5eIE6lLf1m"
   },
   "outputs": [],
   "source": [
    "df=pd.read_csv(\"places.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "cD5i0UFiLf1y",
    "outputId": "b5b46b1f-1088-44e6-bcf6-e6e517e25b0c"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 527 entries, 0 to 526\n",
      "Data columns (total 2 columns):\n",
      "Unnamed: 0    527 non-null int64\n",
      "places        527 non-null object\n",
      "dtypes: int64(1), object(1)\n",
      "memory usage: 6.2+ KB\n"
     ]
    }
   ],
   "source": [
    "df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "UvpKezFMLf2H"
   },
   "outputs": [],
   "source": [
    "df=df.drop(\"Unnamed: 0\",1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "2WAu63e_Lf2T",
    "outputId": "87c7b522-460e-4fba-949e-61e554893863"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 527 entries, 0 to 526\n",
      "Data columns (total 1 columns):\n",
      "places    527 non-null object\n",
      "dtypes: object(1)\n",
      "memory usage: 2.1+ KB\n"
     ]
    }
   ],
   "source": [
    "df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "-j1_8QlvLf2j"
   },
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "tM0EnvJlLf2u"
   },
   "outputs": [],
   "source": [
    "df[\"latitude\"]=np.nan"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "6iddz9GHLf25"
   },
   "outputs": [],
   "source": [
    "df[\"longitude\"]=np.nan"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "D1Dc5PJTLf3E"
   },
   "outputs": [],
   "source": [
    "df[\"class\"]=np.nan"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "BRfwiHliLf3R"
   },
   "outputs": [],
   "source": [
    "df[\"type\"]=np.nan"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "VYFMepHyLf3c",
    "outputId": "3d55ea31-d7a5-45fd-f10d-aed1cb1aedf6"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>places</th>\n",
       "      <th>latitude</th>\n",
       "      <th>longitude</th>\n",
       "      <th>class</th>\n",
       "      <th>type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>KANNURU</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>DODDA BANASHALLI</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>MAHADEVAPURA,</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>BPL LIMITED AVALAHALLI   NDIA</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>AAVALAHALL YELAHANKA</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>AVALAHALLI, OLD MADRAS ROAD, VIRGONAGAR POST</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>11TH MILE, OLD MADRAS ROAD, AVALAHALLI, VIRGON...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>TIMBARHARD LAYOU  AVALAHALLI MYSORE ROAD  26</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>VIRGO NAGAR INDUSTRIAL ESTATE, OPP H.P.PETROL ...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>AVALAHALLI, OLD MADRAS ROAD</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              places  latitude  longitude  \\\n",
       "0                                            KANNURU       NaN        NaN   \n",
       "1                                   DODDA BANASHALLI       NaN        NaN   \n",
       "2                                      MAHADEVAPURA,       NaN        NaN   \n",
       "3                   BPL LIMITED AVALAHALLI   NDIA          NaN        NaN   \n",
       "4                           AAVALAHALL YELAHANKA           NaN        NaN   \n",
       "5       AVALAHALLI, OLD MADRAS ROAD, VIRGONAGAR POST       NaN        NaN   \n",
       "6  11TH MILE, OLD MADRAS ROAD, AVALAHALLI, VIRGON...       NaN        NaN   \n",
       "7    TIMBARHARD LAYOU  AVALAHALLI MYSORE ROAD  26          NaN        NaN   \n",
       "8  VIRGO NAGAR INDUSTRIAL ESTATE, OPP H.P.PETROL ...       NaN        NaN   \n",
       "9                        AVALAHALLI, OLD MADRAS ROAD       NaN        NaN   \n",
       "\n",
       "   class  type  \n",
       "0    NaN   NaN  \n",
       "1    NaN   NaN  \n",
       "2    NaN   NaN  \n",
       "3    NaN   NaN  \n",
       "4    NaN   NaN  \n",
       "5    NaN   NaN  \n",
       "6    NaN   NaN  \n",
       "7    NaN   NaN  \n",
       "8    NaN   NaN  \n",
       "9    NaN   NaN  "
      ]
     },
     "execution_count": 32,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "8jnI_cxcR8Pr"
   },
   "source": [
    "***Nominatim API is used to geocode the location***"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "ux-K6JndLf3v"
   },
   "outputs": [],
   "source": [
    "\n",
    "for index, row in df.iterrows():\n",
    "    text=row['places']\n",
    "    text=' '.join(text.split())\n",
    "    text=text.split(\",\")\n",
    "    text=\"+\".join(text)\n",
    "    text=text.replace(\" \", \"\")\n",
    "    #bounding coordinates added to restrict search results to city of banglore( approx)\n",
    "    client = requests.session()\n",
    "    request = client.get(url='https://nominatim.openstreetmap.org/?format=json&addressdetails=1&q='+text+'&format=json&limit=1&countrycodes=in&bounded=1&viewbox=77.180737,12.732146,77.968935,13.504153',timeout=1)\n",
    "    # variable x is of type string.the size of empty result is 2 ( ie x=[]) \n",
    "    x=request.text\n",
    "    if len(x)!= 2:\n",
    "        parsed_x = json.loads(x)\n",
    "        df.iloc[index,1]=parsed_x[0]['lat']\n",
    "        df.iloc[index,2]=parsed_x[0]['lon']\n",
    "        df.iloc[index,3]=parsed_x[0]['class']\n",
    "        df.iloc[index,4]=parsed_x[0]['type']\n",
    "    \n",
    "    time.sleep(1)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "4qRgNG5iLf35",
    "outputId": "98313745-8979-432b-c9a4-ab6662f1fc28"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>places</th>\n",
       "      <th>latitude</th>\n",
       "      <th>longitude</th>\n",
       "      <th>class</th>\n",
       "      <th>type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>KANNURU</td>\n",
       "      <td>13.0923096</td>\n",
       "      <td>77.6539427</td>\n",
       "      <td>place</td>\n",
       "      <td>village</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>DODDA BANASHALLI</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>MAHADEVAPURA,</td>\n",
       "      <td>13.0671165</td>\n",
       "      <td>77.3201264</td>\n",
       "      <td>place</td>\n",
       "      <td>village</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>BPL LIMITED AVALAHALLI   NDIA</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>AAVALAHALL YELAHANKA</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>AVALAHALLI, OLD MADRAS ROAD, VIRGONAGAR POST</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>11TH MILE, OLD MADRAS ROAD, AVALAHALLI, VIRGON...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>TIMBARHARD LAYOU  AVALAHALLI MYSORE ROAD  26</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>VIRGO NAGAR INDUSTRIAL ESTATE, OPP H.P.PETROL ...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>AVALAHALLI, OLD MADRAS ROAD</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              places    latitude   longitude  \\\n",
       "0                                            KANNURU  13.0923096  77.6539427   \n",
       "1                                   DODDA BANASHALLI         NaN         NaN   \n",
       "2                                      MAHADEVAPURA,  13.0671165  77.3201264   \n",
       "3                   BPL LIMITED AVALAHALLI   NDIA            NaN         NaN   \n",
       "4                           AAVALAHALL YELAHANKA             NaN         NaN   \n",
       "5       AVALAHALLI, OLD MADRAS ROAD, VIRGONAGAR POST         NaN         NaN   \n",
       "6  11TH MILE, OLD MADRAS ROAD, AVALAHALLI, VIRGON...         NaN         NaN   \n",
       "7    TIMBARHARD LAYOU  AVALAHALLI MYSORE ROAD  26            NaN         NaN   \n",
       "8  VIRGO NAGAR INDUSTRIAL ESTATE, OPP H.P.PETROL ...         NaN         NaN   \n",
       "9                        AVALAHALLI, OLD MADRAS ROAD         NaN         NaN   \n",
       "\n",
       "   class     type  \n",
       "0  place  village  \n",
       "1    NaN      NaN  \n",
       "2  place  village  \n",
       "3    NaN      NaN  \n",
       "4    NaN      NaN  \n",
       "5    NaN      NaN  \n",
       "6    NaN      NaN  \n",
       "7    NaN      NaN  \n",
       "8    NaN      NaN  \n",
       "9    NaN      NaN  "
      ]
     },
     "execution_count": 35,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "_59yVWw7SNLs"
   },
   "source": [
    "**Observation**\n",
    "\n",
    "The final result is not satisfactory as the location addresses consisting of spelling mistakes were not classified\n",
    "\n",
    "\n",
    "\n",
    "---\n",
    "\n",
    "\n",
    "\n",
    "**Corrections Needed**\n",
    "\n",
    "*   filter the address based on spelling.\n",
    "*   identify which spelling is found correct by the open street maps\n",
    "*   identify and  create a module for the purpose.\n",
    "\n"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "geocode_v_1.ipynb",
   "provenance": [],
   "version": "0.3.2"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}